ยปCore Development>Code coverage>Lib/test/test_regrtest.py

Python code coverage for Lib/test/test_regrtest.py

#countcontent
1n/a"""
2n/aTests of regrtest.py.
3n/a
4n/aNote: test_regrtest cannot be run twice in parallel.
5n/a"""
6n/a
7n/aimport contextlib
8n/aimport faulthandler
9n/aimport io
10n/aimport os.path
11n/aimport platform
12n/aimport re
13n/aimport subprocess
14n/aimport sys
15n/aimport sysconfig
16n/aimport tempfile
17n/aimport textwrap
18n/aimport unittest
19n/afrom test import libregrtest
20n/afrom test import support
21n/a
22n/a
23n/aPy_DEBUG = hasattr(sys, 'getobjects')
24n/aROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..')
25n/aROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
26n/a
27n/aTEST_INTERRUPTED = textwrap.dedent("""
28n/a from signal import SIGINT
29n/a try:
30n/a from _testcapi import raise_signal
31n/a raise_signal(SIGINT)
32n/a except ImportError:
33n/a import os
34n/a os.kill(os.getpid(), SIGINT)
35n/a """)
36n/a
37n/a
38n/aclass ParseArgsTestCase(unittest.TestCase):
39n/a """
40n/a Test regrtest's argument parsing, function _parse_args().
41n/a """
42n/a
43n/a def checkError(self, args, msg):
44n/a with support.captured_stderr() as err, self.assertRaises(SystemExit):
45n/a libregrtest._parse_args(args)
46n/a self.assertIn(msg, err.getvalue())
47n/a
48n/a def test_help(self):
49n/a for opt in '-h', '--help':
50n/a with self.subTest(opt=opt):
51n/a with support.captured_stdout() as out, \
52n/a self.assertRaises(SystemExit):
53n/a libregrtest._parse_args([opt])
54n/a self.assertIn('Run Python regression tests.', out.getvalue())
55n/a
56n/a @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'),
57n/a "faulthandler.dump_traceback_later() required")
58n/a def test_timeout(self):
59n/a ns = libregrtest._parse_args(['--timeout', '4.2'])
60n/a self.assertEqual(ns.timeout, 4.2)
61n/a self.checkError(['--timeout'], 'expected one argument')
62n/a self.checkError(['--timeout', 'foo'], 'invalid float value')
63n/a
64n/a def test_wait(self):
65n/a ns = libregrtest._parse_args(['--wait'])
66n/a self.assertTrue(ns.wait)
67n/a
68n/a def test_slaveargs(self):
69n/a ns = libregrtest._parse_args(['--slaveargs', '[[], {}]'])
70n/a self.assertEqual(ns.slaveargs, '[[], {}]')
71n/a self.checkError(['--slaveargs'], 'expected one argument')
72n/a
73n/a def test_start(self):
74n/a for opt in '-S', '--start':
75n/a with self.subTest(opt=opt):
76n/a ns = libregrtest._parse_args([opt, 'foo'])
77n/a self.assertEqual(ns.start, 'foo')
78n/a self.checkError([opt], 'expected one argument')
79n/a
80n/a def test_verbose(self):
81n/a ns = libregrtest._parse_args(['-v'])
82n/a self.assertEqual(ns.verbose, 1)
83n/a ns = libregrtest._parse_args(['-vvv'])
84n/a self.assertEqual(ns.verbose, 3)
85n/a ns = libregrtest._parse_args(['--verbose'])
86n/a self.assertEqual(ns.verbose, 1)
87n/a ns = libregrtest._parse_args(['--verbose'] * 3)
88n/a self.assertEqual(ns.verbose, 3)
89n/a ns = libregrtest._parse_args([])
90n/a self.assertEqual(ns.verbose, 0)
91n/a
92n/a def test_verbose2(self):
93n/a for opt in '-w', '--verbose2':
94n/a with self.subTest(opt=opt):
95n/a ns = libregrtest._parse_args([opt])
96n/a self.assertTrue(ns.verbose2)
97n/a
98n/a def test_verbose3(self):
99n/a for opt in '-W', '--verbose3':
100n/a with self.subTest(opt=opt):
101n/a ns = libregrtest._parse_args([opt])
102n/a self.assertTrue(ns.verbose3)
103n/a
104n/a def test_quiet(self):
105n/a for opt in '-q', '--quiet':
106n/a with self.subTest(opt=opt):
107n/a ns = libregrtest._parse_args([opt])
108n/a self.assertTrue(ns.quiet)
109n/a self.assertEqual(ns.verbose, 0)
110n/a
111n/a def test_slow(self):
112n/a for opt in '-o', '--slowest':
113n/a with self.subTest(opt=opt):
114n/a ns = libregrtest._parse_args([opt])
115n/a self.assertTrue(ns.print_slow)
116n/a
117n/a def test_header(self):
118n/a ns = libregrtest._parse_args(['--header'])
119n/a self.assertTrue(ns.header)
120n/a
121n/a def test_randomize(self):
122n/a for opt in '-r', '--randomize':
123n/a with self.subTest(opt=opt):
124n/a ns = libregrtest._parse_args([opt])
125n/a self.assertTrue(ns.randomize)
126n/a
127n/a def test_randseed(self):
128n/a ns = libregrtest._parse_args(['--randseed', '12345'])
129n/a self.assertEqual(ns.random_seed, 12345)
130n/a self.assertTrue(ns.randomize)
131n/a self.checkError(['--randseed'], 'expected one argument')
132n/a self.checkError(['--randseed', 'foo'], 'invalid int value')
133n/a
134n/a def test_fromfile(self):
135n/a for opt in '-f', '--fromfile':
136n/a with self.subTest(opt=opt):
137n/a ns = libregrtest._parse_args([opt, 'foo'])
138n/a self.assertEqual(ns.fromfile, 'foo')
139n/a self.checkError([opt], 'expected one argument')
140n/a self.checkError([opt, 'foo', '-s'], "don't go together")
141n/a
142n/a def test_exclude(self):
143n/a for opt in '-x', '--exclude':
144n/a with self.subTest(opt=opt):
145n/a ns = libregrtest._parse_args([opt])
146n/a self.assertTrue(ns.exclude)
147n/a
148n/a def test_single(self):
149n/a for opt in '-s', '--single':
150n/a with self.subTest(opt=opt):
151n/a ns = libregrtest._parse_args([opt])
152n/a self.assertTrue(ns.single)
153n/a self.checkError([opt, '-f', 'foo'], "don't go together")
154n/a
155n/a def test_match(self):
156n/a for opt in '-m', '--match':
157n/a with self.subTest(opt=opt):
158n/a ns = libregrtest._parse_args([opt, 'pattern'])
159n/a self.assertEqual(ns.match_tests, 'pattern')
160n/a self.checkError([opt], 'expected one argument')
161n/a
162n/a def test_failfast(self):
163n/a for opt in '-G', '--failfast':
164n/a with self.subTest(opt=opt):
165n/a ns = libregrtest._parse_args([opt, '-v'])
166n/a self.assertTrue(ns.failfast)
167n/a ns = libregrtest._parse_args([opt, '-W'])
168n/a self.assertTrue(ns.failfast)
169n/a self.checkError([opt], '-G/--failfast needs either -v or -W')
170n/a
171n/a def test_use(self):
172n/a for opt in '-u', '--use':
173n/a with self.subTest(opt=opt):
174n/a ns = libregrtest._parse_args([opt, 'gui,network'])
175n/a self.assertEqual(ns.use_resources, ['gui', 'network'])
176n/a ns = libregrtest._parse_args([opt, 'gui,none,network'])
177n/a self.assertEqual(ns.use_resources, ['network'])
178n/a expected = list(libregrtest.RESOURCE_NAMES)
179n/a expected.remove('gui')
180n/a ns = libregrtest._parse_args([opt, 'all,-gui'])
181n/a self.assertEqual(ns.use_resources, expected)
182n/a self.checkError([opt], 'expected one argument')
183n/a self.checkError([opt, 'foo'], 'invalid resource')
184n/a
185n/a def test_memlimit(self):
186n/a for opt in '-M', '--memlimit':
187n/a with self.subTest(opt=opt):
188n/a ns = libregrtest._parse_args([opt, '4G'])
189n/a self.assertEqual(ns.memlimit, '4G')
190n/a self.checkError([opt], 'expected one argument')
191n/a
192n/a def test_testdir(self):
193n/a ns = libregrtest._parse_args(['--testdir', 'foo'])
194n/a self.assertEqual(ns.testdir, os.path.join(support.SAVEDCWD, 'foo'))
195n/a self.checkError(['--testdir'], 'expected one argument')
196n/a
197n/a def test_runleaks(self):
198n/a for opt in '-L', '--runleaks':
199n/a with self.subTest(opt=opt):
200n/a ns = libregrtest._parse_args([opt])
201n/a self.assertTrue(ns.runleaks)
202n/a
203n/a def test_huntrleaks(self):
204n/a for opt in '-R', '--huntrleaks':
205n/a with self.subTest(opt=opt):
206n/a ns = libregrtest._parse_args([opt, ':'])
207n/a self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt'))
208n/a ns = libregrtest._parse_args([opt, '6:'])
209n/a self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt'))
210n/a ns = libregrtest._parse_args([opt, ':3'])
211n/a self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt'))
212n/a ns = libregrtest._parse_args([opt, '6:3:leaks.log'])
213n/a self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log'))
214n/a self.checkError([opt], 'expected one argument')
215n/a self.checkError([opt, '6'],
216n/a 'needs 2 or 3 colon-separated arguments')
217n/a self.checkError([opt, 'foo:'], 'invalid huntrleaks value')
218n/a self.checkError([opt, '6:foo'], 'invalid huntrleaks value')
219n/a
220n/a def test_multiprocess(self):
221n/a for opt in '-j', '--multiprocess':
222n/a with self.subTest(opt=opt):
223n/a ns = libregrtest._parse_args([opt, '2'])
224n/a self.assertEqual(ns.use_mp, 2)
225n/a self.checkError([opt], 'expected one argument')
226n/a self.checkError([opt, 'foo'], 'invalid int value')
227n/a self.checkError([opt, '2', '-T'], "don't go together")
228n/a self.checkError([opt, '2', '-l'], "don't go together")
229n/a self.checkError([opt, '0', '-T'], "don't go together")
230n/a self.checkError([opt, '0', '-l'], "don't go together")
231n/a
232n/a def test_coverage(self):
233n/a for opt in '-T', '--coverage':
234n/a with self.subTest(opt=opt):
235n/a ns = libregrtest._parse_args([opt])
236n/a self.assertTrue(ns.trace)
237n/a
238n/a def test_coverdir(self):
239n/a for opt in '-D', '--coverdir':
240n/a with self.subTest(opt=opt):
241n/a ns = libregrtest._parse_args([opt, 'foo'])
242n/a self.assertEqual(ns.coverdir,
243n/a os.path.join(support.SAVEDCWD, 'foo'))
244n/a self.checkError([opt], 'expected one argument')
245n/a
246n/a def test_nocoverdir(self):
247n/a for opt in '-N', '--nocoverdir':
248n/a with self.subTest(opt=opt):
249n/a ns = libregrtest._parse_args([opt])
250n/a self.assertIsNone(ns.coverdir)
251n/a
252n/a def test_threshold(self):
253n/a for opt in '-t', '--threshold':
254n/a with self.subTest(opt=opt):
255n/a ns = libregrtest._parse_args([opt, '1000'])
256n/a self.assertEqual(ns.threshold, 1000)
257n/a self.checkError([opt], 'expected one argument')
258n/a self.checkError([opt, 'foo'], 'invalid int value')
259n/a
260n/a def test_nowindows(self):
261n/a for opt in '-n', '--nowindows':
262n/a with self.subTest(opt=opt):
263n/a with contextlib.redirect_stderr(io.StringIO()) as stderr:
264n/a ns = libregrtest._parse_args([opt])
265n/a self.assertTrue(ns.nowindows)
266n/a err = stderr.getvalue()
267n/a self.assertIn('the --nowindows (-n) option is deprecated', err)
268n/a
269n/a def test_forever(self):
270n/a for opt in '-F', '--forever':
271n/a with self.subTest(opt=opt):
272n/a ns = libregrtest._parse_args([opt])
273n/a self.assertTrue(ns.forever)
274n/a
275n/a
276n/a def test_unrecognized_argument(self):
277n/a self.checkError(['--xxx'], 'usage:')
278n/a
279n/a def test_long_option__partial(self):
280n/a ns = libregrtest._parse_args(['--qui'])
281n/a self.assertTrue(ns.quiet)
282n/a self.assertEqual(ns.verbose, 0)
283n/a
284n/a def test_two_options(self):
285n/a ns = libregrtest._parse_args(['--quiet', '--exclude'])
286n/a self.assertTrue(ns.quiet)
287n/a self.assertEqual(ns.verbose, 0)
288n/a self.assertTrue(ns.exclude)
289n/a
290n/a def test_option_with_empty_string_value(self):
291n/a ns = libregrtest._parse_args(['--start', ''])
292n/a self.assertEqual(ns.start, '')
293n/a
294n/a def test_arg(self):
295n/a ns = libregrtest._parse_args(['foo'])
296n/a self.assertEqual(ns.args, ['foo'])
297n/a
298n/a def test_option_and_arg(self):
299n/a ns = libregrtest._parse_args(['--quiet', 'foo'])
300n/a self.assertTrue(ns.quiet)
301n/a self.assertEqual(ns.verbose, 0)
302n/a self.assertEqual(ns.args, ['foo'])
303n/a
304n/a def test_arg_option_arg(self):
305n/a ns = libregrtest._parse_args(['test_unaryop', '-v', 'test_binop'])
306n/a self.assertEqual(ns.verbose, 1)
307n/a self.assertEqual(ns.args, ['test_unaryop', 'test_binop'])
308n/a
309n/a def test_unknown_option(self):
310n/a self.checkError(['--unknown-option'],
311n/a 'unrecognized arguments: --unknown-option')
312n/a
313n/a
314n/aclass BaseTestCase(unittest.TestCase):
315n/a TEST_UNIQUE_ID = 1
316n/a TESTNAME_PREFIX = 'test_regrtest_'
317n/a TESTNAME_REGEX = r'test_[a-zA-Z0-9_]+'
318n/a
319n/a def setUp(self):
320n/a self.testdir = os.path.realpath(os.path.dirname(__file__))
321n/a
322n/a self.tmptestdir = tempfile.mkdtemp()
323n/a self.addCleanup(support.rmtree, self.tmptestdir)
324n/a
325n/a def create_test(self, name=None, code=''):
326n/a if not name:
327n/a name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID
328n/a BaseTestCase.TEST_UNIQUE_ID += 1
329n/a
330n/a # test_regrtest cannot be run twice in parallel because
331n/a # of setUp() and create_test()
332n/a name = self.TESTNAME_PREFIX + name
333n/a path = os.path.join(self.tmptestdir, name + '.py')
334n/a
335n/a self.addCleanup(support.unlink, path)
336n/a # Use 'x' mode to ensure that we do not override existing tests
337n/a try:
338n/a with open(path, 'x', encoding='utf-8') as fp:
339n/a fp.write(code)
340n/a except PermissionError as exc:
341n/a if not sysconfig.is_python_build():
342n/a self.skipTest("cannot write %s: %s" % (path, exc))
343n/a raise
344n/a return name
345n/a
346n/a def regex_search(self, regex, output):
347n/a match = re.search(regex, output, re.MULTILINE)
348n/a if not match:
349n/a self.fail("%r not found in %r" % (regex, output))
350n/a return match
351n/a
352n/a def check_line(self, output, regex):
353n/a regex = re.compile(r'^' + regex, re.MULTILINE)
354n/a self.assertRegex(output, regex)
355n/a
356n/a def parse_executed_tests(self, output):
357n/a regex = (r'^[0-9]+:[0-9]+:[0-9]+ \[ *[0-9]+(?:/ *[0-9]+)*\] (%s)'
358n/a % self.TESTNAME_REGEX)
359n/a parser = re.finditer(regex, output, re.MULTILINE)
360n/a return list(match.group(1) for match in parser)
361n/a
362n/a def check_executed_tests(self, output, tests, skipped=(), failed=(),
363n/a omitted=(), randomize=False, interrupted=False):
364n/a if isinstance(tests, str):
365n/a tests = [tests]
366n/a if isinstance(skipped, str):
367n/a skipped = [skipped]
368n/a if isinstance(failed, str):
369n/a failed = [failed]
370n/a if isinstance(omitted, str):
371n/a omitted = [omitted]
372n/a ntest = len(tests)
373n/a nskipped = len(skipped)
374n/a nfailed = len(failed)
375n/a nomitted = len(omitted)
376n/a
377n/a executed = self.parse_executed_tests(output)
378n/a if randomize:
379n/a self.assertEqual(set(executed), set(tests), output)
380n/a else:
381n/a self.assertEqual(executed, tests, output)
382n/a
383n/a def plural(count):
384n/a return 's' if count != 1 else ''
385n/a
386n/a def list_regex(line_format, tests):
387n/a count = len(tests)
388n/a names = ' '.join(sorted(tests))
389n/a regex = line_format % (count, plural(count))
390n/a regex = r'%s:\n %s$' % (regex, names)
391n/a return regex
392n/a
393n/a if skipped:
394n/a regex = list_regex('%s test%s skipped', skipped)
395n/a self.check_line(output, regex)
396n/a
397n/a if failed:
398n/a regex = list_regex('%s test%s failed', failed)
399n/a self.check_line(output, regex)
400n/a
401n/a if omitted:
402n/a regex = list_regex('%s test%s omitted', omitted)
403n/a self.check_line(output, regex)
404n/a
405n/a good = ntest - nskipped - nfailed - nomitted
406n/a if good:
407n/a regex = r'%s test%s OK\.$' % (good, plural(good))
408n/a if not skipped and not failed and good > 1:
409n/a regex = 'All %s' % regex
410n/a self.check_line(output, regex)
411n/a
412n/a if interrupted:
413n/a self.check_line(output, 'Test suite interrupted by signal SIGINT.')
414n/a
415n/a if nfailed:
416n/a result = 'FAILURE'
417n/a elif interrupted:
418n/a result = 'INTERRUPTED'
419n/a else:
420n/a result = 'SUCCESS'
421n/a self.check_line(output, 'Tests result: %s' % result)
422n/a
423n/a def parse_random_seed(self, output):
424n/a match = self.regex_search(r'Using random seed ([0-9]+)', output)
425n/a randseed = int(match.group(1))
426n/a self.assertTrue(0 <= randseed <= 10000000, randseed)
427n/a return randseed
428n/a
429n/a def run_command(self, args, input=None, exitcode=0, **kw):
430n/a if not input:
431n/a input = ''
432n/a if 'stderr' not in kw:
433n/a kw['stderr'] = subprocess.PIPE
434n/a proc = subprocess.run(args,
435n/a universal_newlines=True,
436n/a input=input,
437n/a stdout=subprocess.PIPE,
438n/a **kw)
439n/a if proc.returncode != exitcode:
440n/a msg = ("Command %s failed with exit code %s\n"
441n/a "\n"
442n/a "stdout:\n"
443n/a "---\n"
444n/a "%s\n"
445n/a "---\n"
446n/a % (str(args), proc.returncode, proc.stdout))
447n/a if proc.stderr:
448n/a msg += ("\n"
449n/a "stderr:\n"
450n/a "---\n"
451n/a "%s"
452n/a "---\n"
453n/a % proc.stderr)
454n/a self.fail(msg)
455n/a return proc
456n/a
457n/a
458n/a def run_python(self, args, **kw):
459n/a args = [sys.executable, '-X', 'faulthandler', '-I', *args]
460n/a proc = self.run_command(args, **kw)
461n/a return proc.stdout
462n/a
463n/a
464n/aclass ProgramsTestCase(BaseTestCase):
465n/a """
466n/a Test various ways to run the Python test suite. Use options close
467n/a to options used on the buildbot.
468n/a """
469n/a
470n/a NTEST = 4
471n/a
472n/a def setUp(self):
473n/a super().setUp()
474n/a
475n/a # Create NTEST tests doing nothing
476n/a self.tests = [self.create_test() for index in range(self.NTEST)]
477n/a
478n/a self.python_args = ['-Wd', '-E', '-bb']
479n/a self.regrtest_args = ['-uall', '-rwW',
480n/a '--testdir=%s' % self.tmptestdir]
481n/a if hasattr(faulthandler, 'dump_traceback_later'):
482n/a self.regrtest_args.extend(('--timeout', '3600', '-j4'))
483n/a if sys.platform == 'win32':
484n/a self.regrtest_args.append('-n')
485n/a
486n/a def check_output(self, output):
487n/a self.parse_random_seed(output)
488n/a self.check_executed_tests(output, self.tests, randomize=True)
489n/a
490n/a def run_tests(self, args):
491n/a output = self.run_python(args)
492n/a self.check_output(output)
493n/a
494n/a def test_script_regrtest(self):
495n/a # Lib/test/regrtest.py
496n/a script = os.path.join(self.testdir, 'regrtest.py')
497n/a
498n/a args = [*self.python_args, script, *self.regrtest_args, *self.tests]
499n/a self.run_tests(args)
500n/a
501n/a def test_module_test(self):
502n/a # -m test
503n/a args = [*self.python_args, '-m', 'test',
504n/a *self.regrtest_args, *self.tests]
505n/a self.run_tests(args)
506n/a
507n/a def test_module_regrtest(self):
508n/a # -m test.regrtest
509n/a args = [*self.python_args, '-m', 'test.regrtest',
510n/a *self.regrtest_args, *self.tests]
511n/a self.run_tests(args)
512n/a
513n/a def test_module_autotest(self):
514n/a # -m test.autotest
515n/a args = [*self.python_args, '-m', 'test.autotest',
516n/a *self.regrtest_args, *self.tests]
517n/a self.run_tests(args)
518n/a
519n/a def test_module_from_test_autotest(self):
520n/a # from test import autotest
521n/a code = 'from test import autotest'
522n/a args = [*self.python_args, '-c', code,
523n/a *self.regrtest_args, *self.tests]
524n/a self.run_tests(args)
525n/a
526n/a def test_script_autotest(self):
527n/a # Lib/test/autotest.py
528n/a script = os.path.join(self.testdir, 'autotest.py')
529n/a args = [*self.python_args, script, *self.regrtest_args, *self.tests]
530n/a self.run_tests(args)
531n/a
532n/a @unittest.skipUnless(sysconfig.is_python_build(),
533n/a 'run_tests.py script is not installed')
534n/a def test_tools_script_run_tests(self):
535n/a # Tools/scripts/run_tests.py
536n/a script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py')
537n/a args = [script, *self.regrtest_args, *self.tests]
538n/a self.run_tests(args)
539n/a
540n/a def run_batch(self, *args):
541n/a proc = self.run_command(args)
542n/a self.check_output(proc.stdout)
543n/a
544n/a @unittest.skipUnless(sysconfig.is_python_build(),
545n/a 'test.bat script is not installed')
546n/a @unittest.skipUnless(sys.platform == 'win32', 'Windows only')
547n/a def test_tools_buildbot_test(self):
548n/a # Tools\buildbot\test.bat
549n/a script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat')
550n/a test_args = ['--testdir=%s' % self.tmptestdir]
551n/a if platform.architecture()[0] == '64bit':
552n/a test_args.append('-x64') # 64-bit build
553n/a if not Py_DEBUG:
554n/a test_args.append('+d') # Release build, use python.exe
555n/a self.run_batch(script, *test_args, *self.tests)
556n/a
557n/a @unittest.skipUnless(sys.platform == 'win32', 'Windows only')
558n/a def test_pcbuild_rt(self):
559n/a # PCbuild\rt.bat
560n/a script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat')
561n/a rt_args = ["-q"] # Quick, don't run tests twice
562n/a if platform.architecture()[0] == '64bit':
563n/a rt_args.append('-x64') # 64-bit build
564n/a if Py_DEBUG:
565n/a rt_args.append('-d') # Debug build, use python_d.exe
566n/a self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests)
567n/a
568n/a
569n/aclass ArgsTestCase(BaseTestCase):
570n/a """
571n/a Test arguments of the Python test suite.
572n/a """
573n/a
574n/a def run_tests(self, *testargs, **kw):
575n/a cmdargs = ['-m', 'test', '--testdir=%s' % self.tmptestdir, *testargs]
576n/a return self.run_python(cmdargs, **kw)
577n/a
578n/a def test_failing_test(self):
579n/a # test a failing test
580n/a code = textwrap.dedent("""
581n/a import unittest
582n/a
583n/a class FailingTest(unittest.TestCase):
584n/a def test_failing(self):
585n/a self.fail("bug")
586n/a """)
587n/a test_ok = self.create_test('ok')
588n/a test_failing = self.create_test('failing', code=code)
589n/a tests = [test_ok, test_failing]
590n/a
591n/a output = self.run_tests(*tests, exitcode=1)
592n/a self.check_executed_tests(output, tests, failed=test_failing)
593n/a
594n/a def test_resources(self):
595n/a # test -u command line option
596n/a tests = {}
597n/a for resource in ('audio', 'network'):
598n/a code = 'from test import support\nsupport.requires(%r)' % resource
599n/a tests[resource] = self.create_test(resource, code)
600n/a test_names = sorted(tests.values())
601n/a
602n/a # -u all: 2 resources enabled
603n/a output = self.run_tests('-u', 'all', *test_names)
604n/a self.check_executed_tests(output, test_names)
605n/a
606n/a # -u audio: 1 resource enabled
607n/a output = self.run_tests('-uaudio', *test_names)
608n/a self.check_executed_tests(output, test_names,
609n/a skipped=tests['network'])
610n/a
611n/a # no option: 0 resources enabled
612n/a output = self.run_tests(*test_names)
613n/a self.check_executed_tests(output, test_names,
614n/a skipped=test_names)
615n/a
616n/a def test_random(self):
617n/a # test -r and --randseed command line option
618n/a code = textwrap.dedent("""
619n/a import random
620n/a print("TESTRANDOM: %s" % random.randint(1, 1000))
621n/a """)
622n/a test = self.create_test('random', code)
623n/a
624n/a # first run to get the output with the random seed
625n/a output = self.run_tests('-r', test)
626n/a randseed = self.parse_random_seed(output)
627n/a match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
628n/a test_random = int(match.group(1))
629n/a
630n/a # try to reproduce with the random seed
631n/a output = self.run_tests('-r', '--randseed=%s' % randseed, test)
632n/a randseed2 = self.parse_random_seed(output)
633n/a self.assertEqual(randseed2, randseed)
634n/a
635n/a match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
636n/a test_random2 = int(match.group(1))
637n/a self.assertEqual(test_random2, test_random)
638n/a
639n/a def test_fromfile(self):
640n/a # test --fromfile
641n/a tests = [self.create_test() for index in range(5)]
642n/a
643n/a # Write the list of files using a format similar to regrtest output:
644n/a # [1/2] test_1
645n/a # [2/2] test_2
646n/a filename = support.TESTFN
647n/a self.addCleanup(support.unlink, filename)
648n/a
649n/a # test format '0:00:00 [2/7] test_opcodes -- test_grammar took 0 sec'
650n/a with open(filename, "w") as fp:
651n/a previous = None
652n/a for index, name in enumerate(tests, 1):
653n/a line = ("00:00:%02i [%s/%s] %s"
654n/a % (index, index, len(tests), name))
655n/a if previous:
656n/a line += " -- %s took 0 sec" % previous
657n/a print(line, file=fp)
658n/a previous = name
659n/a
660n/a output = self.run_tests('--fromfile', filename)
661n/a self.check_executed_tests(output, tests)
662n/a
663n/a # test format '[2/7] test_opcodes'
664n/a with open(filename, "w") as fp:
665n/a for index, name in enumerate(tests, 1):
666n/a print("[%s/%s] %s" % (index, len(tests), name), file=fp)
667n/a
668n/a output = self.run_tests('--fromfile', filename)
669n/a self.check_executed_tests(output, tests)
670n/a
671n/a # test format 'test_opcodes'
672n/a with open(filename, "w") as fp:
673n/a for name in tests:
674n/a print(name, file=fp)
675n/a
676n/a output = self.run_tests('--fromfile', filename)
677n/a self.check_executed_tests(output, tests)
678n/a
679n/a # test format 'Lib/test/test_opcodes.py'
680n/a with open(filename, "w") as fp:
681n/a for name in tests:
682n/a print('Lib/test/%s.py' % name, file=fp)
683n/a
684n/a output = self.run_tests('--fromfile', filename)
685n/a self.check_executed_tests(output, tests)
686n/a
687n/a def test_interrupted(self):
688n/a code = TEST_INTERRUPTED
689n/a test = self.create_test('sigint', code=code)
690n/a output = self.run_tests(test, exitcode=1)
691n/a self.check_executed_tests(output, test, omitted=test,
692n/a interrupted=True)
693n/a
694n/a def test_slowest(self):
695n/a # test --slowest
696n/a tests = [self.create_test() for index in range(3)]
697n/a output = self.run_tests("--slowest", *tests)
698n/a self.check_executed_tests(output, tests)
699n/a regex = ('10 slowest tests:\n'
700n/a '(?:- %s: .*\n){%s}'
701n/a % (self.TESTNAME_REGEX, len(tests)))
702n/a self.check_line(output, regex)
703n/a
704n/a def test_slow_interrupted(self):
705n/a # Issue #25373: test --slowest with an interrupted test
706n/a code = TEST_INTERRUPTED
707n/a test = self.create_test("sigint", code=code)
708n/a
709n/a try:
710n/a import threading
711n/a tests = (False, True)
712n/a except ImportError:
713n/a tests = (False,)
714n/a for multiprocessing in tests:
715n/a if multiprocessing:
716n/a args = ("--slowest", "-j2", test)
717n/a else:
718n/a args = ("--slowest", test)
719n/a output = self.run_tests(*args, exitcode=1)
720n/a self.check_executed_tests(output, test,
721n/a omitted=test, interrupted=True)
722n/a
723n/a regex = ('10 slowest tests:\n')
724n/a self.check_line(output, regex)
725n/a
726n/a def test_coverage(self):
727n/a # test --coverage
728n/a test = self.create_test('coverage')
729n/a output = self.run_tests("--coverage", test)
730n/a self.check_executed_tests(output, [test])
731n/a regex = (r'lines +cov% +module +\(path\)\n'
732n/a r'(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+')
733n/a self.check_line(output, regex)
734n/a
735n/a def test_wait(self):
736n/a # test --wait
737n/a test = self.create_test('wait')
738n/a output = self.run_tests("--wait", test, input='key')
739n/a self.check_line(output, 'Press any key to continue')
740n/a
741n/a def test_forever(self):
742n/a # test --forever
743n/a code = textwrap.dedent("""
744n/a import builtins
745n/a import unittest
746n/a
747n/a class ForeverTester(unittest.TestCase):
748n/a def test_run(self):
749n/a # Store the state in the builtins module, because the test
750n/a # module is reload at each run
751n/a if 'RUN' in builtins.__dict__:
752n/a builtins.__dict__['RUN'] += 1
753n/a if builtins.__dict__['RUN'] >= 3:
754n/a self.fail("fail at the 3rd runs")
755n/a else:
756n/a builtins.__dict__['RUN'] = 1
757n/a """)
758n/a test = self.create_test('forever', code=code)
759n/a output = self.run_tests('--forever', test, exitcode=1)
760n/a self.check_executed_tests(output, [test]*3, failed=test)
761n/a
762n/a @unittest.skipUnless(Py_DEBUG, 'need a debug build')
763n/a def test_huntrleaks_fd_leak(self):
764n/a # test --huntrleaks for file descriptor leak
765n/a code = textwrap.dedent("""
766n/a import os
767n/a import unittest
768n/a
769n/a # Issue #25306: Disable popups and logs to stderr on assertion
770n/a # failures in MSCRT
771n/a try:
772n/a import msvcrt
773n/a msvcrt.CrtSetReportMode
774n/a except (ImportError, AttributeError):
775n/a # no Windows, o release build
776n/a pass
777n/a else:
778n/a for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]:
779n/a msvcrt.CrtSetReportMode(m, 0)
780n/a
781n/a class FDLeakTest(unittest.TestCase):
782n/a def test_leak(self):
783n/a fd = os.open(__file__, os.O_RDONLY)
784n/a # bug: never cloes the file descriptor
785n/a """)
786n/a test = self.create_test('huntrleaks', code=code)
787n/a
788n/a filename = 'reflog.txt'
789n/a self.addCleanup(support.unlink, filename)
790n/a output = self.run_tests('--huntrleaks', '3:3:', test,
791n/a exitcode=1,
792n/a stderr=subprocess.STDOUT)
793n/a self.check_executed_tests(output, [test], failed=test)
794n/a
795n/a line = 'beginning 6 repetitions\n123456\n......\n'
796n/a self.check_line(output, re.escape(line))
797n/a
798n/a line2 = '%s leaked [1, 1, 1] file descriptors, sum=3\n' % test
799n/a self.assertIn(line2, output)
800n/a
801n/a with open(filename) as fp:
802n/a reflog = fp.read()
803n/a self.assertIn(line2, reflog)
804n/a
805n/a def test_list_tests(self):
806n/a # test --list-tests
807n/a tests = [self.create_test() for i in range(5)]
808n/a output = self.run_tests('--list-tests', *tests)
809n/a self.assertEqual(output.rstrip().splitlines(),
810n/a tests)
811n/a
812n/a def test_crashed(self):
813n/a # Any code which causes a crash
814n/a code = 'import faulthandler; faulthandler._sigsegv()'
815n/a crash_test = self.create_test(name="crash", code=code)
816n/a ok_test = self.create_test(name="ok")
817n/a
818n/a tests = [crash_test, ok_test]
819n/a output = self.run_tests("-j2", *tests, exitcode=1)
820n/a self.check_executed_tests(output, tests, failed=crash_test,
821n/a randomize=True)
822n/a
823n/a
824n/aif __name__ == '__main__':
825n/a unittest.main()