Python code coverage for Lib/test/test_pickle.py

#	count	content
1	n/a	from _compat_pickle import (IMPORT_MAPPING, REVERSE_IMPORT_MAPPING,
2	n/a	NAME_MAPPING, REVERSE_NAME_MAPPING)
3	n/a	import builtins
4	n/a	import pickle
5	n/a	import io
6	n/a	import collections
7	n/a	import struct
8	n/a	import sys
9	n/a
10	n/a	import unittest
11	n/a	from test import support
12	n/a
13	n/a	from test.pickletester import AbstractUnpickleTests
14	n/a	from test.pickletester import AbstractPickleTests
15	n/a	from test.pickletester import AbstractPickleModuleTests
16	n/a	from test.pickletester import AbstractPersistentPicklerTests
17	n/a	from test.pickletester import AbstractIdentityPersistentPicklerTests
18	n/a	from test.pickletester import AbstractPicklerUnpicklerObjectTests
19	n/a	from test.pickletester import AbstractDispatchTableTests
20	n/a	from test.pickletester import BigmemPickleTests
21	n/a
22	n/a	try:
23	n/a	import _pickle
24	n/a	has_c_implementation = True
25	n/a	except ImportError:
26	n/a	has_c_implementation = False
27	n/a
28	n/a
29	n/a	class PickleTests(AbstractPickleModuleTests):
30	n/a	pass
31	n/a
32	n/a
33	n/a	class PyUnpicklerTests(AbstractUnpickleTests):
34	n/a
35	n/a	unpickler = pickle._Unpickler
36	n/a	bad_stack_errors = (IndexError,)
37	n/a	truncated_errors = (pickle.UnpicklingError, EOFError,
38	n/a	AttributeError, ValueError,
39	n/a	struct.error, IndexError, ImportError)
40	n/a
41	n/a	def loads(self, buf, **kwds):
42	n/a	f = io.BytesIO(buf)
43	n/a	u = self.unpickler(f, **kwds)
44	n/a	return u.load()
45	n/a
46	n/a
47	n/a	class PyPicklerTests(AbstractPickleTests):
48	n/a
49	n/a	pickler = pickle._Pickler
50	n/a	unpickler = pickle._Unpickler
51	n/a
52	n/a	def dumps(self, arg, proto=None):
53	n/a	f = io.BytesIO()
54	n/a	p = self.pickler(f, proto)
55	n/a	p.dump(arg)
56	n/a	f.seek(0)
57	n/a	return bytes(f.read())
58	n/a
59	n/a	def loads(self, buf, **kwds):
60	n/a	f = io.BytesIO(buf)
61	n/a	u = self.unpickler(f, **kwds)
62	n/a	return u.load()
63	n/a
64	n/a
65	n/a	class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests,
66	n/a	BigmemPickleTests):
67	n/a
68	n/a	pickler = pickle._Pickler
69	n/a	unpickler = pickle._Unpickler
70	n/a	bad_stack_errors = (pickle.UnpicklingError, IndexError)
71	n/a	truncated_errors = (pickle.UnpicklingError, EOFError,
72	n/a	AttributeError, ValueError,
73	n/a	struct.error, IndexError, ImportError)
74	n/a
75	n/a	def dumps(self, arg, protocol=None):
76	n/a	return pickle.dumps(arg, protocol)
77	n/a
78	n/a	def loads(self, buf, **kwds):
79	n/a	return pickle.loads(buf, **kwds)
80	n/a
81	n/a
82	n/a	class PersistentPicklerUnpicklerMixin(object):
83	n/a
84	n/a	def dumps(self, arg, proto=None):
85	n/a	class PersPickler(self.pickler):
86	n/a	def persistent_id(subself, obj):
87	n/a	return self.persistent_id(obj)
88	n/a	f = io.BytesIO()
89	n/a	p = PersPickler(f, proto)
90	n/a	p.dump(arg)
91	n/a	return f.getvalue()
92	n/a
93	n/a	def loads(self, buf, **kwds):
94	n/a	class PersUnpickler(self.unpickler):
95	n/a	def persistent_load(subself, obj):
96	n/a	return self.persistent_load(obj)
97	n/a	f = io.BytesIO(buf)
98	n/a	u = PersUnpickler(f, **kwds)
99	n/a	return u.load()
100	n/a
101	n/a
102	n/a	class PyPersPicklerTests(AbstractPersistentPicklerTests,
103	n/a	PersistentPicklerUnpicklerMixin):
104	n/a
105	n/a	pickler = pickle._Pickler
106	n/a	unpickler = pickle._Unpickler
107	n/a
108	n/a
109	n/a	class PyIdPersPicklerTests(AbstractIdentityPersistentPicklerTests,
110	n/a	PersistentPicklerUnpicklerMixin):
111	n/a
112	n/a	pickler = pickle._Pickler
113	n/a	unpickler = pickle._Unpickler
114	n/a
115	n/a
116	n/a	class PyPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests):
117	n/a
118	n/a	pickler_class = pickle._Pickler
119	n/a	unpickler_class = pickle._Unpickler
120	n/a
121	n/a
122	n/a	class PyDispatchTableTests(AbstractDispatchTableTests):
123	n/a
124	n/a	pickler_class = pickle._Pickler
125	n/a
126	n/a	def get_dispatch_table(self):
127	n/a	return pickle.dispatch_table.copy()
128	n/a
129	n/a
130	n/a	class PyChainDispatchTableTests(AbstractDispatchTableTests):
131	n/a
132	n/a	pickler_class = pickle._Pickler
133	n/a
134	n/a	def get_dispatch_table(self):
135	n/a	return collections.ChainMap({}, pickle.dispatch_table)
136	n/a
137	n/a
138	n/a	if has_c_implementation:
139	n/a	class CUnpicklerTests(PyUnpicklerTests):
140	n/a	unpickler = _pickle.Unpickler
141	n/a	bad_stack_errors = (pickle.UnpicklingError,)
142	n/a	truncated_errors = (pickle.UnpicklingError,)
143	n/a
144	n/a	class CPicklerTests(PyPicklerTests):
145	n/a	pickler = _pickle.Pickler
146	n/a	unpickler = _pickle.Unpickler
147	n/a
148	n/a	class CPersPicklerTests(PyPersPicklerTests):
149	n/a	pickler = _pickle.Pickler
150	n/a	unpickler = _pickle.Unpickler
151	n/a
152	n/a	class CIdPersPicklerTests(PyIdPersPicklerTests):
153	n/a	pickler = _pickle.Pickler
154	n/a	unpickler = _pickle.Unpickler
155	n/a
156	n/a	class CDumpPickle_LoadPickle(PyPicklerTests):
157	n/a	pickler = _pickle.Pickler
158	n/a	unpickler = pickle._Unpickler
159	n/a
160	n/a	class DumpPickle_CLoadPickle(PyPicklerTests):
161	n/a	pickler = pickle._Pickler
162	n/a	unpickler = _pickle.Unpickler
163	n/a
164	n/a	class CPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests):
165	n/a	pickler_class = _pickle.Pickler
166	n/a	unpickler_class = _pickle.Unpickler
167	n/a
168	n/a	def test_issue18339(self):
169	n/a	unpickler = self.unpickler_class(io.BytesIO())
170	n/a	with self.assertRaises(TypeError):
171	n/a	unpickler.memo = object
172	n/a	# used to cause a segfault
173	n/a	with self.assertRaises(ValueError):
174	n/a	unpickler.memo = {-1: None}
175	n/a	unpickler.memo = {1: None}
176	n/a
177	n/a	class CDispatchTableTests(AbstractDispatchTableTests):
178	n/a	pickler_class = pickle.Pickler
179	n/a	def get_dispatch_table(self):
180	n/a	return pickle.dispatch_table.copy()
181	n/a
182	n/a	class CChainDispatchTableTests(AbstractDispatchTableTests):
183	n/a	pickler_class = pickle.Pickler
184	n/a	def get_dispatch_table(self):
185	n/a	return collections.ChainMap({}, pickle.dispatch_table)
186	n/a
187	n/a	@support.cpython_only
188	n/a	class SizeofTests(unittest.TestCase):
189	n/a	check_sizeof = support.check_sizeof
190	n/a
191	n/a	def test_pickler(self):
192	n/a	basesize = support.calcobjsize('5P2n3i2n3iP')
193	n/a	p = _pickle.Pickler(io.BytesIO())
194	n/a	self.assertEqual(object.__sizeof__(p), basesize)
195	n/a	MT_size = struct.calcsize('3nP0n')
196	n/a	ME_size = struct.calcsize('Pn0P')
197	n/a	check = self.check_sizeof
198	n/a	check(p, basesize +
199	n/a	MT_size + 8 * ME_size + # Minimal memo table size.
200	n/a	sys.getsizeof(b'x'*4096)) # Minimal write buffer size.
201	n/a	for i in range(6):
202	n/a	p.dump(chr(i))
203	n/a	check(p, basesize +
204	n/a	MT_size + 32 * ME_size + # Size of memo table required to
205	n/a	# save references to 6 objects.
206	n/a	0) # Write buffer is cleared after every dump().
207	n/a
208	n/a	def test_unpickler(self):
209	n/a	basesize = support.calcobjsize('2Pn2P 2P2n2i5P 2P3n6P2n2i')
210	n/a	unpickler = _pickle.Unpickler
211	n/a	P = struct.calcsize('P') # Size of memo table entry.
212	n/a	n = struct.calcsize('n') # Size of mark table entry.
213	n/a	check = self.check_sizeof
214	n/a	for encoding in 'ASCII', 'UTF-16', 'latin-1':
215	n/a	for errors in 'strict', 'replace':
216	n/a	u = unpickler(io.BytesIO(),
217	n/a	encoding=encoding, errors=errors)
218	n/a	self.assertEqual(object.__sizeof__(u), basesize)
219	n/a	check(u, basesize +
220	n/a	32 * P + # Minimal memo table size.
221	n/a	len(encoding) + 1 + len(errors) + 1)
222	n/a
223	n/a	stdsize = basesize + len('ASCII') + 1 + len('strict') + 1
224	n/a	def check_unpickler(data, memo_size, marks_size):
225	n/a	dump = pickle.dumps(data)
226	n/a	u = unpickler(io.BytesIO(dump),
227	n/a	encoding='ASCII', errors='strict')
228	n/a	u.load()
229	n/a	check(u, stdsize + memo_size * P + marks_size * n)
230	n/a
231	n/a	check_unpickler(0, 32, 0)
232	n/a	# 20 is minimal non-empty mark stack size.
233	n/a	check_unpickler([0] * 100, 32, 20)
234	n/a	# 128 is memo table size required to save references to 100 objects.
235	n/a	check_unpickler([chr(i) for i in range(100)], 128, 20)
236	n/a	def recurse(deep):
237	n/a	data = 0
238	n/a	for i in range(deep):
239	n/a	data = [data, data]
240	n/a	return data
241	n/a	check_unpickler(recurse(0), 32, 0)
242	n/a	check_unpickler(recurse(1), 32, 20)
243	n/a	check_unpickler(recurse(20), 32, 58)
244	n/a	check_unpickler(recurse(50), 64, 58)
245	n/a	check_unpickler(recurse(100), 128, 134)
246	n/a
247	n/a	u = unpickler(io.BytesIO(pickle.dumps('a', 0)),
248	n/a	encoding='ASCII', errors='strict')
249	n/a	u.load()
250	n/a	check(u, stdsize + 32 * P + 2 + 1)
251	n/a
252	n/a
253	n/a	ALT_IMPORT_MAPPING = {
254	n/a	('_elementtree', 'xml.etree.ElementTree'),
255	n/a	('cPickle', 'pickle'),
256	n/a	('StringIO', 'io'),
257	n/a	('cStringIO', 'io'),
258	n/a	}
259	n/a
260	n/a	ALT_NAME_MAPPING = {
261	n/a	('__builtin__', 'basestring', 'builtins', 'str'),
262	n/a	('exceptions', 'StandardError', 'builtins', 'Exception'),
263	n/a	('UserDict', 'UserDict', 'collections', 'UserDict'),
264	n/a	('socket', '_socketobject', 'socket', 'SocketType'),
265	n/a	}
266	n/a
267	n/a	def mapping(module, name):
268	n/a	if (module, name) in NAME_MAPPING:
269	n/a	module, name = NAME_MAPPING[(module, name)]
270	n/a	elif module in IMPORT_MAPPING:
271	n/a	module = IMPORT_MAPPING[module]
272	n/a	return module, name
273	n/a
274	n/a	def reverse_mapping(module, name):
275	n/a	if (module, name) in REVERSE_NAME_MAPPING:
276	n/a	module, name = REVERSE_NAME_MAPPING[(module, name)]
277	n/a	elif module in REVERSE_IMPORT_MAPPING:
278	n/a	module = REVERSE_IMPORT_MAPPING[module]
279	n/a	return module, name
280	n/a
281	n/a	def getmodule(module):
282	n/a	try:
283	n/a	return sys.modules[module]
284	n/a	except KeyError:
285	n/a	try:
286	n/a	__import__(module)
287	n/a	except AttributeError as exc:
288	n/a	if support.verbose:
289	n/a	print("Can't import module %r: %s" % (module, exc))
290	n/a	raise ImportError
291	n/a	except ImportError as exc:
292	n/a	if support.verbose:
293	n/a	print(exc)
294	n/a	raise
295	n/a	return sys.modules[module]
296	n/a
297	n/a	def getattribute(module, name):
298	n/a	obj = getmodule(module)
299	n/a	for n in name.split('.'):
300	n/a	obj = getattr(obj, n)
301	n/a	return obj
302	n/a
303	n/a	def get_exceptions(mod):
304	n/a	for name in dir(mod):
305	n/a	attr = getattr(mod, name)
306	n/a	if isinstance(attr, type) and issubclass(attr, BaseException):
307	n/a	yield name, attr
308	n/a
309	n/a	class CompatPickleTests(unittest.TestCase):
310	n/a	def test_import(self):
311	n/a	modules = set(IMPORT_MAPPING.values())
312	n/a	modules \|= set(REVERSE_IMPORT_MAPPING)
313	n/a	modules \|= {module for module, name in REVERSE_NAME_MAPPING}
314	n/a	modules \|= {module for module, name in NAME_MAPPING.values()}
315	n/a	for module in modules:
316	n/a	try:
317	n/a	getmodule(module)
318	n/a	except ImportError:
319	n/a	pass
320	n/a
321	n/a	def test_import_mapping(self):
322	n/a	for module3, module2 in REVERSE_IMPORT_MAPPING.items():
323	n/a	with self.subTest((module3, module2)):
324	n/a	try:
325	n/a	getmodule(module3)
326	n/a	except ImportError:
327	n/a	pass
328	n/a	if module3[:1] != '_':
329	n/a	self.assertIn(module2, IMPORT_MAPPING)
330	n/a	self.assertEqual(IMPORT_MAPPING[module2], module3)
331	n/a
332	n/a	def test_name_mapping(self):
333	n/a	for (module3, name3), (module2, name2) in REVERSE_NAME_MAPPING.items():
334	n/a	with self.subTest(((module3, name3), (module2, name2))):
335	n/a	if (module2, name2) == ('exceptions', 'OSError'):
336	n/a	attr = getattribute(module3, name3)
337	n/a	self.assertTrue(issubclass(attr, OSError))
338	n/a	elif (module2, name2) == ('exceptions', 'ImportError'):
339	n/a	attr = getattribute(module3, name3)
340	n/a	self.assertTrue(issubclass(attr, ImportError))
341	n/a	else:
342	n/a	module, name = mapping(module2, name2)
343	n/a	if module3[:1] != '_':
344	n/a	self.assertEqual((module, name), (module3, name3))
345	n/a	try:
346	n/a	attr = getattribute(module3, name3)
347	n/a	except ImportError:
348	n/a	pass
349	n/a	else:
350	n/a	self.assertEqual(getattribute(module, name), attr)
351	n/a
352	n/a	def test_reverse_import_mapping(self):
353	n/a	for module2, module3 in IMPORT_MAPPING.items():
354	n/a	with self.subTest((module2, module3)):
355	n/a	try:
356	n/a	getmodule(module3)
357	n/a	except ImportError as exc:
358	n/a	if support.verbose:
359	n/a	print(exc)
360	n/a	if ((module2, module3) not in ALT_IMPORT_MAPPING and
361	n/a	REVERSE_IMPORT_MAPPING.get(module3, None) != module2):
362	n/a	for (m3, n3), (m2, n2) in REVERSE_NAME_MAPPING.items():
363	n/a	if (module3, module2) == (m3, m2):
364	n/a	break
365	n/a	else:
366	n/a	self.fail('No reverse mapping from %r to %r' %
367	n/a	(module3, module2))
368	n/a	module = REVERSE_IMPORT_MAPPING.get(module3, module3)
369	n/a	module = IMPORT_MAPPING.get(module, module)
370	n/a	self.assertEqual(module, module3)
371	n/a
372	n/a	def test_reverse_name_mapping(self):
373	n/a	for (module2, name2), (module3, name3) in NAME_MAPPING.items():
374	n/a	with self.subTest(((module2, name2), (module3, name3))):
375	n/a	try:
376	n/a	attr = getattribute(module3, name3)
377	n/a	except ImportError:
378	n/a	pass
379	n/a	module, name = reverse_mapping(module3, name3)
380	n/a	if (module2, name2, module3, name3) not in ALT_NAME_MAPPING:
381	n/a	self.assertEqual((module, name), (module2, name2))
382	n/a	module, name = mapping(module, name)
383	n/a	self.assertEqual((module, name), (module3, name3))
384	n/a
385	n/a	def test_exceptions(self):
386	n/a	self.assertEqual(mapping('exceptions', 'StandardError'),
387	n/a	('builtins', 'Exception'))
388	n/a	self.assertEqual(mapping('exceptions', 'Exception'),
389	n/a	('builtins', 'Exception'))
390	n/a	self.assertEqual(reverse_mapping('builtins', 'Exception'),
391	n/a	('exceptions', 'Exception'))
392	n/a	self.assertEqual(mapping('exceptions', 'OSError'),
393	n/a	('builtins', 'OSError'))
394	n/a	self.assertEqual(reverse_mapping('builtins', 'OSError'),
395	n/a	('exceptions', 'OSError'))
396	n/a
397	n/a	for name, exc in get_exceptions(builtins):
398	n/a	with self.subTest(name):
399	n/a	if exc in (BlockingIOError,
400	n/a	ResourceWarning,
401	n/a	StopAsyncIteration,
402	n/a	RecursionError):
403	n/a	continue
404	n/a	if exc is not OSError and issubclass(exc, OSError):
405	n/a	self.assertEqual(reverse_mapping('builtins', name),
406	n/a	('exceptions', 'OSError'))
407	n/a	elif exc is not ImportError and issubclass(exc, ImportError):
408	n/a	self.assertEqual(reverse_mapping('builtins', name),
409	n/a	('exceptions', 'ImportError'))
410	n/a	self.assertEqual(mapping('exceptions', name),
411	n/a	('exceptions', name))
412	n/a	else:
413	n/a	self.assertEqual(reverse_mapping('builtins', name),
414	n/a	('exceptions', name))
415	n/a	self.assertEqual(mapping('exceptions', name),
416	n/a	('builtins', name))
417	n/a
418	n/a	def test_multiprocessing_exceptions(self):
419	n/a	module = support.import_module('multiprocessing.context')
420	n/a	for name, exc in get_exceptions(module):
421	n/a	with self.subTest(name):
422	n/a	self.assertEqual(reverse_mapping('multiprocessing.context', name),
423	n/a	('multiprocessing', name))
424	n/a	self.assertEqual(mapping('multiprocessing', name),
425	n/a	('multiprocessing.context', name))
426	n/a
427	n/a
428	n/a	def test_main():
429	n/a	tests = [PickleTests, PyUnpicklerTests, PyPicklerTests,
430	n/a	PyPersPicklerTests, PyIdPersPicklerTests,
431	n/a	PyDispatchTableTests, PyChainDispatchTableTests,
432	n/a	CompatPickleTests]
433	n/a	if has_c_implementation:
434	n/a	tests.extend([CUnpicklerTests, CPicklerTests,
435	n/a	CPersPicklerTests, CIdPersPicklerTests,
436	n/a	CDumpPickle_LoadPickle, DumpPickle_CLoadPickle,
437	n/a	PyPicklerUnpicklerObjectTests,
438	n/a	CPicklerUnpicklerObjectTests,
439	n/a	CDispatchTableTests, CChainDispatchTableTests,
440	n/a	InMemoryPickleTests, SizeofTests])
441	n/a	support.run_unittest(*tests)
442	n/a	support.run_doctest(pickle)
443	n/a
444	n/a	if __name__ == "__main__":
445	n/a	test_main()