ยปCore Development>Code coverage>Lib/pickle.py

Python code coverage for Lib/pickle.py

#countcontent
1n/a"""Create portable serialized representations of Python objects.
2n/a
3n/aSee module copyreg for a mechanism for registering custom picklers.
4n/aSee module pickletools source for extensive comments.
5n/a
6n/aClasses:
7n/a
8n/a Pickler
9n/a Unpickler
10n/a
11n/aFunctions:
12n/a
13n/a dump(object, file)
14n/a dumps(object) -> string
15n/a load(file) -> object
16n/a loads(string) -> object
17n/a
18n/aMisc variables:
19n/a
20n/a __version__
21n/a format_version
22n/a compatible_formats
23n/a
24n/a"""
25n/a
26n/afrom types import FunctionType
27n/afrom copyreg import dispatch_table
28n/afrom copyreg import _extension_registry, _inverted_registry, _extension_cache
29n/afrom itertools import islice
30n/afrom functools import partial
31n/aimport sys
32n/afrom sys import maxsize
33n/afrom struct import pack, unpack
34n/aimport re
35n/aimport io
36n/aimport codecs
37n/aimport _compat_pickle
38n/a
39n/a__all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler",
40n/a "Unpickler", "dump", "dumps", "load", "loads"]
41n/a
42n/a# Shortcut for use in isinstance testing
43n/abytes_types = (bytes, bytearray)
44n/a
45n/a# These are purely informational; no code uses these.
46n/aformat_version = "4.0" # File format version we write
47n/acompatible_formats = ["1.0", # Original protocol 0
48n/a "1.1", # Protocol 0 with INST added
49n/a "1.2", # Original protocol 1
50n/a "1.3", # Protocol 1 with BINFLOAT added
51n/a "2.0", # Protocol 2
52n/a "3.0", # Protocol 3
53n/a "4.0", # Protocol 4
54n/a ] # Old format versions we can read
55n/a
56n/a# This is the highest protocol number we know how to read.
57n/aHIGHEST_PROTOCOL = 4
58n/a
59n/a# The protocol we write by default. May be less than HIGHEST_PROTOCOL.
60n/a# We intentionally write a protocol that Python 2.x cannot read;
61n/a# there are too many issues with that.
62n/aDEFAULT_PROTOCOL = 3
63n/a
64n/aclass PickleError(Exception):
65n/a """A common base class for the other pickling exceptions."""
66n/a pass
67n/a
68n/aclass PicklingError(PickleError):
69n/a """This exception is raised when an unpicklable object is passed to the
70n/a dump() method.
71n/a
72n/a """
73n/a pass
74n/a
75n/aclass UnpicklingError(PickleError):
76n/a """This exception is raised when there is a problem unpickling an object,
77n/a such as a security violation.
78n/a
79n/a Note that other exceptions may also be raised during unpickling, including
80n/a (but not necessarily limited to) AttributeError, EOFError, ImportError,
81n/a and IndexError.
82n/a
83n/a """
84n/a pass
85n/a
86n/a# An instance of _Stop is raised by Unpickler.load_stop() in response to
87n/a# the STOP opcode, passing the object that is the result of unpickling.
88n/aclass _Stop(Exception):
89n/a def __init__(self, value):
90n/a self.value = value
91n/a
92n/a# Jython has PyStringMap; it's a dict subclass with string keys
93n/atry:
94n/a from org.python.core import PyStringMap
95n/aexcept ImportError:
96n/a PyStringMap = None
97n/a
98n/a# Pickle opcodes. See pickletools.py for extensive docs. The listing
99n/a# here is in kind-of alphabetical order of 1-character pickle code.
100n/a# pickletools groups them by purpose.
101n/a
102n/aMARK = b'(' # push special markobject on stack
103n/aSTOP = b'.' # every pickle ends with STOP
104n/aPOP = b'0' # discard topmost stack item
105n/aPOP_MARK = b'1' # discard stack top through topmost markobject
106n/aDUP = b'2' # duplicate top stack item
107n/aFLOAT = b'F' # push float object; decimal string argument
108n/aINT = b'I' # push integer or bool; decimal string argument
109n/aBININT = b'J' # push four-byte signed int
110n/aBININT1 = b'K' # push 1-byte unsigned int
111n/aLONG = b'L' # push long; decimal string argument
112n/aBININT2 = b'M' # push 2-byte unsigned int
113n/aNONE = b'N' # push None
114n/aPERSID = b'P' # push persistent object; id is taken from string arg
115n/aBINPERSID = b'Q' # " " " ; " " " " stack
116n/aREDUCE = b'R' # apply callable to argtuple, both on stack
117n/aSTRING = b'S' # push string; NL-terminated string argument
118n/aBINSTRING = b'T' # push string; counted binary string argument
119n/aSHORT_BINSTRING= b'U' # " " ; " " " " < 256 bytes
120n/aUNICODE = b'V' # push Unicode string; raw-unicode-escaped'd argument
121n/aBINUNICODE = b'X' # " " " ; counted UTF-8 string argument
122n/aAPPEND = b'a' # append stack top to list below it
123n/aBUILD = b'b' # call __setstate__ or __dict__.update()
124n/aGLOBAL = b'c' # push self.find_class(modname, name); 2 string args
125n/aDICT = b'd' # build a dict from stack items
126n/aEMPTY_DICT = b'}' # push empty dict
127n/aAPPENDS = b'e' # extend list on stack by topmost stack slice
128n/aGET = b'g' # push item from memo on stack; index is string arg
129n/aBINGET = b'h' # " " " " " " ; " " 1-byte arg
130n/aINST = b'i' # build & push class instance
131n/aLONG_BINGET = b'j' # push item from memo on stack; index is 4-byte arg
132n/aLIST = b'l' # build list from topmost stack items
133n/aEMPTY_LIST = b']' # push empty list
134n/aOBJ = b'o' # build & push class instance
135n/aPUT = b'p' # store stack top in memo; index is string arg
136n/aBINPUT = b'q' # " " " " " ; " " 1-byte arg
137n/aLONG_BINPUT = b'r' # " " " " " ; " " 4-byte arg
138n/aSETITEM = b's' # add key+value pair to dict
139n/aTUPLE = b't' # build tuple from topmost stack items
140n/aEMPTY_TUPLE = b')' # push empty tuple
141n/aSETITEMS = b'u' # modify dict by adding topmost key+value pairs
142n/aBINFLOAT = b'G' # push float; arg is 8-byte float encoding
143n/a
144n/aTRUE = b'I01\n' # not an opcode; see INT docs in pickletools.py
145n/aFALSE = b'I00\n' # not an opcode; see INT docs in pickletools.py
146n/a
147n/a# Protocol 2
148n/a
149n/aPROTO = b'\x80' # identify pickle protocol
150n/aNEWOBJ = b'\x81' # build object by applying cls.__new__ to argtuple
151n/aEXT1 = b'\x82' # push object from extension registry; 1-byte index
152n/aEXT2 = b'\x83' # ditto, but 2-byte index
153n/aEXT4 = b'\x84' # ditto, but 4-byte index
154n/aTUPLE1 = b'\x85' # build 1-tuple from stack top
155n/aTUPLE2 = b'\x86' # build 2-tuple from two topmost stack items
156n/aTUPLE3 = b'\x87' # build 3-tuple from three topmost stack items
157n/aNEWTRUE = b'\x88' # push True
158n/aNEWFALSE = b'\x89' # push False
159n/aLONG1 = b'\x8a' # push long from < 256 bytes
160n/aLONG4 = b'\x8b' # push really big long
161n/a
162n/a_tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]
163n/a
164n/a# Protocol 3 (Python 3.x)
165n/a
166n/aBINBYTES = b'B' # push bytes; counted binary string argument
167n/aSHORT_BINBYTES = b'C' # " " ; " " " " < 256 bytes
168n/a
169n/a# Protocol 4
170n/aSHORT_BINUNICODE = b'\x8c' # push short string; UTF-8 length < 256 bytes
171n/aBINUNICODE8 = b'\x8d' # push very long string
172n/aBINBYTES8 = b'\x8e' # push very long bytes string
173n/aEMPTY_SET = b'\x8f' # push empty set on the stack
174n/aADDITEMS = b'\x90' # modify set by adding topmost stack items
175n/aFROZENSET = b'\x91' # build frozenset from topmost stack items
176n/aNEWOBJ_EX = b'\x92' # like NEWOBJ but work with keyword only arguments
177n/aSTACK_GLOBAL = b'\x93' # same as GLOBAL but using names on the stacks
178n/aMEMOIZE = b'\x94' # store top of the stack in memo
179n/aFRAME = b'\x95' # indicate the beginning of a new frame
180n/a
181n/a__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)])
182n/a
183n/a
184n/aclass _Framer:
185n/a
186n/a _FRAME_SIZE_TARGET = 64 * 1024
187n/a
188n/a def __init__(self, file_write):
189n/a self.file_write = file_write
190n/a self.current_frame = None
191n/a
192n/a def start_framing(self):
193n/a self.current_frame = io.BytesIO()
194n/a
195n/a def end_framing(self):
196n/a if self.current_frame and self.current_frame.tell() > 0:
197n/a self.commit_frame(force=True)
198n/a self.current_frame = None
199n/a
200n/a def commit_frame(self, force=False):
201n/a if self.current_frame:
202n/a f = self.current_frame
203n/a if f.tell() >= self._FRAME_SIZE_TARGET or force:
204n/a with f.getbuffer() as data:
205n/a n = len(data)
206n/a write = self.file_write
207n/a write(FRAME)
208n/a write(pack("<Q", n))
209n/a write(data)
210n/a f.seek(0)
211n/a f.truncate()
212n/a
213n/a def write(self, data):
214n/a if self.current_frame:
215n/a return self.current_frame.write(data)
216n/a else:
217n/a return self.file_write(data)
218n/a
219n/a
220n/aclass _Unframer:
221n/a
222n/a def __init__(self, file_read, file_readline, file_tell=None):
223n/a self.file_read = file_read
224n/a self.file_readline = file_readline
225n/a self.current_frame = None
226n/a
227n/a def read(self, n):
228n/a if self.current_frame:
229n/a data = self.current_frame.read(n)
230n/a if not data and n != 0:
231n/a self.current_frame = None
232n/a return self.file_read(n)
233n/a if len(data) < n:
234n/a raise UnpicklingError(
235n/a "pickle exhausted before end of frame")
236n/a return data
237n/a else:
238n/a return self.file_read(n)
239n/a
240n/a def readline(self):
241n/a if self.current_frame:
242n/a data = self.current_frame.readline()
243n/a if not data:
244n/a self.current_frame = None
245n/a return self.file_readline()
246n/a if data[-1] != b'\n'[0]:
247n/a raise UnpicklingError(
248n/a "pickle exhausted before end of frame")
249n/a return data
250n/a else:
251n/a return self.file_readline()
252n/a
253n/a def load_frame(self, frame_size):
254n/a if self.current_frame and self.current_frame.read() != b'':
255n/a raise UnpicklingError(
256n/a "beginning of a new frame before end of current frame")
257n/a self.current_frame = io.BytesIO(self.file_read(frame_size))
258n/a
259n/a
260n/a# Tools used for pickling.
261n/a
262n/adef _getattribute(obj, name):
263n/a for subpath in name.split('.'):
264n/a if subpath == '<locals>':
265n/a raise AttributeError("Can't get local attribute {!r} on {!r}"
266n/a .format(name, obj))
267n/a try:
268n/a parent = obj
269n/a obj = getattr(obj, subpath)
270n/a except AttributeError:
271n/a raise AttributeError("Can't get attribute {!r} on {!r}"
272n/a .format(name, obj))
273n/a return obj, parent
274n/a
275n/adef whichmodule(obj, name):
276n/a """Find the module an object belong to."""
277n/a module_name = getattr(obj, '__module__', None)
278n/a if module_name is not None:
279n/a return module_name
280n/a # Protect the iteration by using a list copy of sys.modules against dynamic
281n/a # modules that trigger imports of other modules upon calls to getattr.
282n/a for module_name, module in list(sys.modules.items()):
283n/a if module_name == '__main__' or module is None:
284n/a continue
285n/a try:
286n/a if _getattribute(module, name)[0] is obj:
287n/a return module_name
288n/a except AttributeError:
289n/a pass
290n/a return '__main__'
291n/a
292n/adef encode_long(x):
293n/a r"""Encode a long to a two's complement little-endian binary string.
294n/a Note that 0 is a special case, returning an empty string, to save a
295n/a byte in the LONG1 pickling context.
296n/a
297n/a >>> encode_long(0)
298n/a b''
299n/a >>> encode_long(255)
300n/a b'\xff\x00'
301n/a >>> encode_long(32767)
302n/a b'\xff\x7f'
303n/a >>> encode_long(-256)
304n/a b'\x00\xff'
305n/a >>> encode_long(-32768)
306n/a b'\x00\x80'
307n/a >>> encode_long(-128)
308n/a b'\x80'
309n/a >>> encode_long(127)
310n/a b'\x7f'
311n/a >>>
312n/a """
313n/a if x == 0:
314n/a return b''
315n/a nbytes = (x.bit_length() >> 3) + 1
316n/a result = x.to_bytes(nbytes, byteorder='little', signed=True)
317n/a if x < 0 and nbytes > 1:
318n/a if result[-1] == 0xff and (result[-2] & 0x80) != 0:
319n/a result = result[:-1]
320n/a return result
321n/a
322n/adef decode_long(data):
323n/a r"""Decode a long from a two's complement little-endian binary string.
324n/a
325n/a >>> decode_long(b'')
326n/a 0
327n/a >>> decode_long(b"\xff\x00")
328n/a 255
329n/a >>> decode_long(b"\xff\x7f")
330n/a 32767
331n/a >>> decode_long(b"\x00\xff")
332n/a -256
333n/a >>> decode_long(b"\x00\x80")
334n/a -32768
335n/a >>> decode_long(b"\x80")
336n/a -128
337n/a >>> decode_long(b"\x7f")
338n/a 127
339n/a """
340n/a return int.from_bytes(data, byteorder='little', signed=True)
341n/a
342n/a
343n/a# Pickling machinery
344n/a
345n/aclass _Pickler:
346n/a
347n/a def __init__(self, file, protocol=None, *, fix_imports=True):
348n/a """This takes a binary file for writing a pickle data stream.
349n/a
350n/a The optional *protocol* argument tells the pickler to use the
351n/a given protocol; supported protocols are 0, 1, 2, 3 and 4. The
352n/a default protocol is 3; a backward-incompatible protocol designed
353n/a for Python 3.
354n/a
355n/a Specifying a negative protocol version selects the highest
356n/a protocol version supported. The higher the protocol used, the
357n/a more recent the version of Python needed to read the pickle
358n/a produced.
359n/a
360n/a The *file* argument must have a write() method that accepts a
361n/a single bytes argument. It can thus be a file object opened for
362n/a binary writing, an io.BytesIO instance, or any other custom
363n/a object that meets this interface.
364n/a
365n/a If *fix_imports* is True and *protocol* is less than 3, pickle
366n/a will try to map the new Python 3 names to the old module names
367n/a used in Python 2, so that the pickle data stream is readable
368n/a with Python 2.
369n/a """
370n/a if protocol is None:
371n/a protocol = DEFAULT_PROTOCOL
372n/a if protocol < 0:
373n/a protocol = HIGHEST_PROTOCOL
374n/a elif not 0 <= protocol <= HIGHEST_PROTOCOL:
375n/a raise ValueError("pickle protocol must be <= %d" % HIGHEST_PROTOCOL)
376n/a try:
377n/a self._file_write = file.write
378n/a except AttributeError:
379n/a raise TypeError("file must have a 'write' attribute")
380n/a self.framer = _Framer(self._file_write)
381n/a self.write = self.framer.write
382n/a self.memo = {}
383n/a self.proto = int(protocol)
384n/a self.bin = protocol >= 1
385n/a self.fast = 0
386n/a self.fix_imports = fix_imports and protocol < 3
387n/a
388n/a def clear_memo(self):
389n/a """Clears the pickler's "memo".
390n/a
391n/a The memo is the data structure that remembers which objects the
392n/a pickler has already seen, so that shared or recursive objects
393n/a are pickled by reference and not by value. This method is
394n/a useful when re-using picklers.
395n/a """
396n/a self.memo.clear()
397n/a
398n/a def dump(self, obj):
399n/a """Write a pickled representation of obj to the open file."""
400n/a # Check whether Pickler was initialized correctly. This is
401n/a # only needed to mimic the behavior of _pickle.Pickler.dump().
402n/a if not hasattr(self, "_file_write"):
403n/a raise PicklingError("Pickler.__init__() was not called by "
404n/a "%s.__init__()" % (self.__class__.__name__,))
405n/a if self.proto >= 2:
406n/a self.write(PROTO + pack("<B", self.proto))
407n/a if self.proto >= 4:
408n/a self.framer.start_framing()
409n/a self.save(obj)
410n/a self.write(STOP)
411n/a self.framer.end_framing()
412n/a
413n/a def memoize(self, obj):
414n/a """Store an object in the memo."""
415n/a
416n/a # The Pickler memo is a dictionary mapping object ids to 2-tuples
417n/a # that contain the Unpickler memo key and the object being memoized.
418n/a # The memo key is written to the pickle and will become
419n/a # the key in the Unpickler's memo. The object is stored in the
420n/a # Pickler memo so that transient objects are kept alive during
421n/a # pickling.
422n/a
423n/a # The use of the Unpickler memo length as the memo key is just a
424n/a # convention. The only requirement is that the memo values be unique.
425n/a # But there appears no advantage to any other scheme, and this
426n/a # scheme allows the Unpickler memo to be implemented as a plain (but
427n/a # growable) array, indexed by memo key.
428n/a if self.fast:
429n/a return
430n/a assert id(obj) not in self.memo
431n/a idx = len(self.memo)
432n/a self.write(self.put(idx))
433n/a self.memo[id(obj)] = idx, obj
434n/a
435n/a # Return a PUT (BINPUT, LONG_BINPUT) opcode string, with argument i.
436n/a def put(self, idx):
437n/a if self.proto >= 4:
438n/a return MEMOIZE
439n/a elif self.bin:
440n/a if idx < 256:
441n/a return BINPUT + pack("<B", idx)
442n/a else:
443n/a return LONG_BINPUT + pack("<I", idx)
444n/a else:
445n/a return PUT + repr(idx).encode("ascii") + b'\n'
446n/a
447n/a # Return a GET (BINGET, LONG_BINGET) opcode string, with argument i.
448n/a def get(self, i):
449n/a if self.bin:
450n/a if i < 256:
451n/a return BINGET + pack("<B", i)
452n/a else:
453n/a return LONG_BINGET + pack("<I", i)
454n/a
455n/a return GET + repr(i).encode("ascii") + b'\n'
456n/a
457n/a def save(self, obj, save_persistent_id=True):
458n/a self.framer.commit_frame()
459n/a
460n/a # Check for persistent id (defined by a subclass)
461n/a pid = self.persistent_id(obj)
462n/a if pid is not None and save_persistent_id:
463n/a self.save_pers(pid)
464n/a return
465n/a
466n/a # Check the memo
467n/a x = self.memo.get(id(obj))
468n/a if x is not None:
469n/a self.write(self.get(x[0]))
470n/a return
471n/a
472n/a # Check the type dispatch table
473n/a t = type(obj)
474n/a f = self.dispatch.get(t)
475n/a if f is not None:
476n/a f(self, obj) # Call unbound method with explicit self
477n/a return
478n/a
479n/a # Check private dispatch table if any, or else copyreg.dispatch_table
480n/a reduce = getattr(self, 'dispatch_table', dispatch_table).get(t)
481n/a if reduce is not None:
482n/a rv = reduce(obj)
483n/a else:
484n/a # Check for a class with a custom metaclass; treat as regular class
485n/a try:
486n/a issc = issubclass(t, type)
487n/a except TypeError: # t is not a class (old Boost; see SF #502085)
488n/a issc = False
489n/a if issc:
490n/a self.save_global(obj)
491n/a return
492n/a
493n/a # Check for a __reduce_ex__ method, fall back to __reduce__
494n/a reduce = getattr(obj, "__reduce_ex__", None)
495n/a if reduce is not None:
496n/a rv = reduce(self.proto)
497n/a else:
498n/a reduce = getattr(obj, "__reduce__", None)
499n/a if reduce is not None:
500n/a rv = reduce()
501n/a else:
502n/a raise PicklingError("Can't pickle %r object: %r" %
503n/a (t.__name__, obj))
504n/a
505n/a # Check for string returned by reduce(), meaning "save as global"
506n/a if isinstance(rv, str):
507n/a self.save_global(obj, rv)
508n/a return
509n/a
510n/a # Assert that reduce() returned a tuple
511n/a if not isinstance(rv, tuple):
512n/a raise PicklingError("%s must return string or tuple" % reduce)
513n/a
514n/a # Assert that it returned an appropriately sized tuple
515n/a l = len(rv)
516n/a if not (2 <= l <= 5):
517n/a raise PicklingError("Tuple returned by %s must have "
518n/a "two to five elements" % reduce)
519n/a
520n/a # Save the reduce() output and finally memoize the object
521n/a self.save_reduce(obj=obj, *rv)
522n/a
523n/a def persistent_id(self, obj):
524n/a # This exists so a subclass can override it
525n/a return None
526n/a
527n/a def save_pers(self, pid):
528n/a # Save a persistent id reference
529n/a if self.bin:
530n/a self.save(pid, save_persistent_id=False)
531n/a self.write(BINPERSID)
532n/a else:
533n/a try:
534n/a self.write(PERSID + str(pid).encode("ascii") + b'\n')
535n/a except UnicodeEncodeError:
536n/a raise PicklingError(
537n/a "persistent IDs in protocol 0 must be ASCII strings")
538n/a
539n/a def save_reduce(self, func, args, state=None, listitems=None,
540n/a dictitems=None, obj=None):
541n/a # This API is called by some subclasses
542n/a
543n/a if not isinstance(args, tuple):
544n/a raise PicklingError("args from save_reduce() must be a tuple")
545n/a if not callable(func):
546n/a raise PicklingError("func from save_reduce() must be callable")
547n/a
548n/a save = self.save
549n/a write = self.write
550n/a
551n/a func_name = getattr(func, "__name__", "")
552n/a if self.proto >= 2 and func_name == "__newobj_ex__":
553n/a cls, args, kwargs = args
554n/a if not hasattr(cls, "__new__"):
555n/a raise PicklingError("args[0] from {} args has no __new__"
556n/a .format(func_name))
557n/a if obj is not None and cls is not obj.__class__:
558n/a raise PicklingError("args[0] from {} args has the wrong class"
559n/a .format(func_name))
560n/a if self.proto >= 4:
561n/a save(cls)
562n/a save(args)
563n/a save(kwargs)
564n/a write(NEWOBJ_EX)
565n/a else:
566n/a func = partial(cls.__new__, cls, *args, **kwargs)
567n/a save(func)
568n/a save(())
569n/a write(REDUCE)
570n/a elif self.proto >= 2 and func_name == "__newobj__":
571n/a # A __reduce__ implementation can direct protocol 2 or newer to
572n/a # use the more efficient NEWOBJ opcode, while still
573n/a # allowing protocol 0 and 1 to work normally. For this to
574n/a # work, the function returned by __reduce__ should be
575n/a # called __newobj__, and its first argument should be a
576n/a # class. The implementation for __newobj__
577n/a # should be as follows, although pickle has no way to
578n/a # verify this:
579n/a #
580n/a # def __newobj__(cls, *args):
581n/a # return cls.__new__(cls, *args)
582n/a #
583n/a # Protocols 0 and 1 will pickle a reference to __newobj__,
584n/a # while protocol 2 (and above) will pickle a reference to
585n/a # cls, the remaining args tuple, and the NEWOBJ code,
586n/a # which calls cls.__new__(cls, *args) at unpickling time
587n/a # (see load_newobj below). If __reduce__ returns a
588n/a # three-tuple, the state from the third tuple item will be
589n/a # pickled regardless of the protocol, calling __setstate__
590n/a # at unpickling time (see load_build below).
591n/a #
592n/a # Note that no standard __newobj__ implementation exists;
593n/a # you have to provide your own. This is to enforce
594n/a # compatibility with Python 2.2 (pickles written using
595n/a # protocol 0 or 1 in Python 2.3 should be unpicklable by
596n/a # Python 2.2).
597n/a cls = args[0]
598n/a if not hasattr(cls, "__new__"):
599n/a raise PicklingError(
600n/a "args[0] from __newobj__ args has no __new__")
601n/a if obj is not None and cls is not obj.__class__:
602n/a raise PicklingError(
603n/a "args[0] from __newobj__ args has the wrong class")
604n/a args = args[1:]
605n/a save(cls)
606n/a save(args)
607n/a write(NEWOBJ)
608n/a else:
609n/a save(func)
610n/a save(args)
611n/a write(REDUCE)
612n/a
613n/a if obj is not None:
614n/a # If the object is already in the memo, this means it is
615n/a # recursive. In this case, throw away everything we put on the
616n/a # stack, and fetch the object back from the memo.
617n/a if id(obj) in self.memo:
618n/a write(POP + self.get(self.memo[id(obj)][0]))
619n/a else:
620n/a self.memoize(obj)
621n/a
622n/a # More new special cases (that work with older protocols as
623n/a # well): when __reduce__ returns a tuple with 4 or 5 items,
624n/a # the 4th and 5th item should be iterators that provide list
625n/a # items and dict items (as (key, value) tuples), or None.
626n/a
627n/a if listitems is not None:
628n/a self._batch_appends(listitems)
629n/a
630n/a if dictitems is not None:
631n/a self._batch_setitems(dictitems)
632n/a
633n/a if state is not None:
634n/a save(state)
635n/a write(BUILD)
636n/a
637n/a # Methods below this point are dispatched through the dispatch table
638n/a
639n/a dispatch = {}
640n/a
641n/a def save_none(self, obj):
642n/a self.write(NONE)
643n/a dispatch[type(None)] = save_none
644n/a
645n/a def save_bool(self, obj):
646n/a if self.proto >= 2:
647n/a self.write(NEWTRUE if obj else NEWFALSE)
648n/a else:
649n/a self.write(TRUE if obj else FALSE)
650n/a dispatch[bool] = save_bool
651n/a
652n/a def save_long(self, obj):
653n/a if self.bin:
654n/a # If the int is small enough to fit in a signed 4-byte 2's-comp
655n/a # format, we can store it more efficiently than the general
656n/a # case.
657n/a # First one- and two-byte unsigned ints:
658n/a if obj >= 0:
659n/a if obj <= 0xff:
660n/a self.write(BININT1 + pack("<B", obj))
661n/a return
662n/a if obj <= 0xffff:
663n/a self.write(BININT2 + pack("<H", obj))
664n/a return
665n/a # Next check for 4-byte signed ints:
666n/a if -0x80000000 <= obj <= 0x7fffffff:
667n/a self.write(BININT + pack("<i", obj))
668n/a return
669n/a if self.proto >= 2:
670n/a encoded = encode_long(obj)
671n/a n = len(encoded)
672n/a if n < 256:
673n/a self.write(LONG1 + pack("<B", n) + encoded)
674n/a else:
675n/a self.write(LONG4 + pack("<i", n) + encoded)
676n/a return
677n/a self.write(LONG + repr(obj).encode("ascii") + b'L\n')
678n/a dispatch[int] = save_long
679n/a
680n/a def save_float(self, obj):
681n/a if self.bin:
682n/a self.write(BINFLOAT + pack('>d', obj))
683n/a else:
684n/a self.write(FLOAT + repr(obj).encode("ascii") + b'\n')
685n/a dispatch[float] = save_float
686n/a
687n/a def save_bytes(self, obj):
688n/a if self.proto < 3:
689n/a if not obj: # bytes object is empty
690n/a self.save_reduce(bytes, (), obj=obj)
691n/a else:
692n/a self.save_reduce(codecs.encode,
693n/a (str(obj, 'latin1'), 'latin1'), obj=obj)
694n/a return
695n/a n = len(obj)
696n/a if n <= 0xff:
697n/a self.write(SHORT_BINBYTES + pack("<B", n) + obj)
698n/a elif n > 0xffffffff and self.proto >= 4:
699n/a self.write(BINBYTES8 + pack("<Q", n) + obj)
700n/a else:
701n/a self.write(BINBYTES + pack("<I", n) + obj)
702n/a self.memoize(obj)
703n/a dispatch[bytes] = save_bytes
704n/a
705n/a def save_str(self, obj):
706n/a if self.bin:
707n/a encoded = obj.encode('utf-8', 'surrogatepass')
708n/a n = len(encoded)
709n/a if n <= 0xff and self.proto >= 4:
710n/a self.write(SHORT_BINUNICODE + pack("<B", n) + encoded)
711n/a elif n > 0xffffffff and self.proto >= 4:
712n/a self.write(BINUNICODE8 + pack("<Q", n) + encoded)
713n/a else:
714n/a self.write(BINUNICODE + pack("<I", n) + encoded)
715n/a else:
716n/a obj = obj.replace("\\", "\\u005c")
717n/a obj = obj.replace("\n", "\\u000a")
718n/a self.write(UNICODE + obj.encode('raw-unicode-escape') +
719n/a b'\n')
720n/a self.memoize(obj)
721n/a dispatch[str] = save_str
722n/a
723n/a def save_tuple(self, obj):
724n/a if not obj: # tuple is empty
725n/a if self.bin:
726n/a self.write(EMPTY_TUPLE)
727n/a else:
728n/a self.write(MARK + TUPLE)
729n/a return
730n/a
731n/a n = len(obj)
732n/a save = self.save
733n/a memo = self.memo
734n/a if n <= 3 and self.proto >= 2:
735n/a for element in obj:
736n/a save(element)
737n/a # Subtle. Same as in the big comment below.
738n/a if id(obj) in memo:
739n/a get = self.get(memo[id(obj)][0])
740n/a self.write(POP * n + get)
741n/a else:
742n/a self.write(_tuplesize2code[n])
743n/a self.memoize(obj)
744n/a return
745n/a
746n/a # proto 0 or proto 1 and tuple isn't empty, or proto > 1 and tuple
747n/a # has more than 3 elements.
748n/a write = self.write
749n/a write(MARK)
750n/a for element in obj:
751n/a save(element)
752n/a
753n/a if id(obj) in memo:
754n/a # Subtle. d was not in memo when we entered save_tuple(), so
755n/a # the process of saving the tuple's elements must have saved
756n/a # the tuple itself: the tuple is recursive. The proper action
757n/a # now is to throw away everything we put on the stack, and
758n/a # simply GET the tuple (it's already constructed). This check
759n/a # could have been done in the "for element" loop instead, but
760n/a # recursive tuples are a rare thing.
761n/a get = self.get(memo[id(obj)][0])
762n/a if self.bin:
763n/a write(POP_MARK + get)
764n/a else: # proto 0 -- POP_MARK not available
765n/a write(POP * (n+1) + get)
766n/a return
767n/a
768n/a # No recursion.
769n/a write(TUPLE)
770n/a self.memoize(obj)
771n/a
772n/a dispatch[tuple] = save_tuple
773n/a
774n/a def save_list(self, obj):
775n/a if self.bin:
776n/a self.write(EMPTY_LIST)
777n/a else: # proto 0 -- can't use EMPTY_LIST
778n/a self.write(MARK + LIST)
779n/a
780n/a self.memoize(obj)
781n/a self._batch_appends(obj)
782n/a
783n/a dispatch[list] = save_list
784n/a
785n/a _BATCHSIZE = 1000
786n/a
787n/a def _batch_appends(self, items):
788n/a # Helper to batch up APPENDS sequences
789n/a save = self.save
790n/a write = self.write
791n/a
792n/a if not self.bin:
793n/a for x in items:
794n/a save(x)
795n/a write(APPEND)
796n/a return
797n/a
798n/a it = iter(items)
799n/a while True:
800n/a tmp = list(islice(it, self._BATCHSIZE))
801n/a n = len(tmp)
802n/a if n > 1:
803n/a write(MARK)
804n/a for x in tmp:
805n/a save(x)
806n/a write(APPENDS)
807n/a elif n:
808n/a save(tmp[0])
809n/a write(APPEND)
810n/a # else tmp is empty, and we're done
811n/a if n < self._BATCHSIZE:
812n/a return
813n/a
814n/a def save_dict(self, obj):
815n/a if self.bin:
816n/a self.write(EMPTY_DICT)
817n/a else: # proto 0 -- can't use EMPTY_DICT
818n/a self.write(MARK + DICT)
819n/a
820n/a self.memoize(obj)
821n/a self._batch_setitems(obj.items())
822n/a
823n/a dispatch[dict] = save_dict
824n/a if PyStringMap is not None:
825n/a dispatch[PyStringMap] = save_dict
826n/a
827n/a def _batch_setitems(self, items):
828n/a # Helper to batch up SETITEMS sequences; proto >= 1 only
829n/a save = self.save
830n/a write = self.write
831n/a
832n/a if not self.bin:
833n/a for k, v in items:
834n/a save(k)
835n/a save(v)
836n/a write(SETITEM)
837n/a return
838n/a
839n/a it = iter(items)
840n/a while True:
841n/a tmp = list(islice(it, self._BATCHSIZE))
842n/a n = len(tmp)
843n/a if n > 1:
844n/a write(MARK)
845n/a for k, v in tmp:
846n/a save(k)
847n/a save(v)
848n/a write(SETITEMS)
849n/a elif n:
850n/a k, v = tmp[0]
851n/a save(k)
852n/a save(v)
853n/a write(SETITEM)
854n/a # else tmp is empty, and we're done
855n/a if n < self._BATCHSIZE:
856n/a return
857n/a
858n/a def save_set(self, obj):
859n/a save = self.save
860n/a write = self.write
861n/a
862n/a if self.proto < 4:
863n/a self.save_reduce(set, (list(obj),), obj=obj)
864n/a return
865n/a
866n/a write(EMPTY_SET)
867n/a self.memoize(obj)
868n/a
869n/a it = iter(obj)
870n/a while True:
871n/a batch = list(islice(it, self._BATCHSIZE))
872n/a n = len(batch)
873n/a if n > 0:
874n/a write(MARK)
875n/a for item in batch:
876n/a save(item)
877n/a write(ADDITEMS)
878n/a if n < self._BATCHSIZE:
879n/a return
880n/a dispatch[set] = save_set
881n/a
882n/a def save_frozenset(self, obj):
883n/a save = self.save
884n/a write = self.write
885n/a
886n/a if self.proto < 4:
887n/a self.save_reduce(frozenset, (list(obj),), obj=obj)
888n/a return
889n/a
890n/a write(MARK)
891n/a for item in obj:
892n/a save(item)
893n/a
894n/a if id(obj) in self.memo:
895n/a # If the object is already in the memo, this means it is
896n/a # recursive. In this case, throw away everything we put on the
897n/a # stack, and fetch the object back from the memo.
898n/a write(POP_MARK + self.get(self.memo[id(obj)][0]))
899n/a return
900n/a
901n/a write(FROZENSET)
902n/a self.memoize(obj)
903n/a dispatch[frozenset] = save_frozenset
904n/a
905n/a def save_global(self, obj, name=None):
906n/a write = self.write
907n/a memo = self.memo
908n/a
909n/a if name is None:
910n/a name = getattr(obj, '__qualname__', None)
911n/a if name is None:
912n/a name = obj.__name__
913n/a
914n/a module_name = whichmodule(obj, name)
915n/a try:
916n/a __import__(module_name, level=0)
917n/a module = sys.modules[module_name]
918n/a obj2, parent = _getattribute(module, name)
919n/a except (ImportError, KeyError, AttributeError):
920n/a raise PicklingError(
921n/a "Can't pickle %r: it's not found as %s.%s" %
922n/a (obj, module_name, name))
923n/a else:
924n/a if obj2 is not obj:
925n/a raise PicklingError(
926n/a "Can't pickle %r: it's not the same object as %s.%s" %
927n/a (obj, module_name, name))
928n/a
929n/a if self.proto >= 2:
930n/a code = _extension_registry.get((module_name, name))
931n/a if code:
932n/a assert code > 0
933n/a if code <= 0xff:
934n/a write(EXT1 + pack("<B", code))
935n/a elif code <= 0xffff:
936n/a write(EXT2 + pack("<H", code))
937n/a else:
938n/a write(EXT4 + pack("<i", code))
939n/a return
940n/a lastname = name.rpartition('.')[2]
941n/a if parent is module:
942n/a name = lastname
943n/a # Non-ASCII identifiers are supported only with protocols >= 3.
944n/a if self.proto >= 4:
945n/a self.save(module_name)
946n/a self.save(name)
947n/a write(STACK_GLOBAL)
948n/a elif parent is not module:
949n/a self.save_reduce(getattr, (parent, lastname))
950n/a elif self.proto >= 3:
951n/a write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
952n/a bytes(name, "utf-8") + b'\n')
953n/a else:
954n/a if self.fix_imports:
955n/a r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
956n/a r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
957n/a if (module_name, name) in r_name_mapping:
958n/a module_name, name = r_name_mapping[(module_name, name)]
959n/a elif module_name in r_import_mapping:
960n/a module_name = r_import_mapping[module_name]
961n/a try:
962n/a write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
963n/a bytes(name, "ascii") + b'\n')
964n/a except UnicodeEncodeError:
965n/a raise PicklingError(
966n/a "can't pickle global identifier '%s.%s' using "
967n/a "pickle protocol %i" % (module, name, self.proto))
968n/a
969n/a self.memoize(obj)
970n/a
971n/a def save_type(self, obj):
972n/a if obj is type(None):
973n/a return self.save_reduce(type, (None,), obj=obj)
974n/a elif obj is type(NotImplemented):
975n/a return self.save_reduce(type, (NotImplemented,), obj=obj)
976n/a elif obj is type(...):
977n/a return self.save_reduce(type, (...,), obj=obj)
978n/a return self.save_global(obj)
979n/a
980n/a dispatch[FunctionType] = save_global
981n/a dispatch[type] = save_type
982n/a
983n/a
984n/a# Unpickling machinery
985n/a
986n/aclass _Unpickler:
987n/a
988n/a def __init__(self, file, *, fix_imports=True,
989n/a encoding="ASCII", errors="strict"):
990n/a """This takes a binary file for reading a pickle data stream.
991n/a
992n/a The protocol version of the pickle is detected automatically, so
993n/a no proto argument is needed.
994n/a
995n/a The argument *file* must have two methods, a read() method that
996n/a takes an integer argument, and a readline() method that requires
997n/a no arguments. Both methods should return bytes. Thus *file*
998n/a can be a binary file object opened for reading, an io.BytesIO
999n/a object, or any other custom object that meets this interface.
1000n/a
1001n/a The file-like object must have two methods, a read() method
1002n/a that takes an integer argument, and a readline() method that
1003n/a requires no arguments. Both methods should return bytes.
1004n/a Thus file-like object can be a binary file object opened for
1005n/a reading, a BytesIO object, or any other custom object that
1006n/a meets this interface.
1007n/a
1008n/a Optional keyword arguments are *fix_imports*, *encoding* and
1009n/a *errors*, which are used to control compatibility support for
1010n/a pickle stream generated by Python 2. If *fix_imports* is True,
1011n/a pickle will try to map the old Python 2 names to the new names
1012n/a used in Python 3. The *encoding* and *errors* tell pickle how
1013n/a to decode 8-bit string instances pickled by Python 2; these
1014n/a default to 'ASCII' and 'strict', respectively. *encoding* can be
1015n/a 'bytes' to read theses 8-bit string instances as bytes objects.
1016n/a """
1017n/a self._file_readline = file.readline
1018n/a self._file_read = file.read
1019n/a self.memo = {}
1020n/a self.encoding = encoding
1021n/a self.errors = errors
1022n/a self.proto = 0
1023n/a self.fix_imports = fix_imports
1024n/a
1025n/a def load(self):
1026n/a """Read a pickled object representation from the open file.
1027n/a
1028n/a Return the reconstituted object hierarchy specified in the file.
1029n/a """
1030n/a # Check whether Unpickler was initialized correctly. This is
1031n/a # only needed to mimic the behavior of _pickle.Unpickler.dump().
1032n/a if not hasattr(self, "_file_read"):
1033n/a raise UnpicklingError("Unpickler.__init__() was not called by "
1034n/a "%s.__init__()" % (self.__class__.__name__,))
1035n/a self._unframer = _Unframer(self._file_read, self._file_readline)
1036n/a self.read = self._unframer.read
1037n/a self.readline = self._unframer.readline
1038n/a self.metastack = []
1039n/a self.stack = []
1040n/a self.append = self.stack.append
1041n/a self.proto = 0
1042n/a read = self.read
1043n/a dispatch = self.dispatch
1044n/a try:
1045n/a while True:
1046n/a key = read(1)
1047n/a if not key:
1048n/a raise EOFError
1049n/a assert isinstance(key, bytes_types)
1050n/a dispatch[key[0]](self)
1051n/a except _Stop as stopinst:
1052n/a return stopinst.value
1053n/a
1054n/a # Return a list of items pushed in the stack after last MARK instruction.
1055n/a def pop_mark(self):
1056n/a items = self.stack
1057n/a self.stack = self.metastack.pop()
1058n/a self.append = self.stack.append
1059n/a return items
1060n/a
1061n/a def persistent_load(self, pid):
1062n/a raise UnpicklingError("unsupported persistent id encountered")
1063n/a
1064n/a dispatch = {}
1065n/a
1066n/a def load_proto(self):
1067n/a proto = self.read(1)[0]
1068n/a if not 0 <= proto <= HIGHEST_PROTOCOL:
1069n/a raise ValueError("unsupported pickle protocol: %d" % proto)
1070n/a self.proto = proto
1071n/a dispatch[PROTO[0]] = load_proto
1072n/a
1073n/a def load_frame(self):
1074n/a frame_size, = unpack('<Q', self.read(8))
1075n/a if frame_size > sys.maxsize:
1076n/a raise ValueError("frame size > sys.maxsize: %d" % frame_size)
1077n/a self._unframer.load_frame(frame_size)
1078n/a dispatch[FRAME[0]] = load_frame
1079n/a
1080n/a def load_persid(self):
1081n/a try:
1082n/a pid = self.readline()[:-1].decode("ascii")
1083n/a except UnicodeDecodeError:
1084n/a raise UnpicklingError(
1085n/a "persistent IDs in protocol 0 must be ASCII strings")
1086n/a self.append(self.persistent_load(pid))
1087n/a dispatch[PERSID[0]] = load_persid
1088n/a
1089n/a def load_binpersid(self):
1090n/a pid = self.stack.pop()
1091n/a self.append(self.persistent_load(pid))
1092n/a dispatch[BINPERSID[0]] = load_binpersid
1093n/a
1094n/a def load_none(self):
1095n/a self.append(None)
1096n/a dispatch[NONE[0]] = load_none
1097n/a
1098n/a def load_false(self):
1099n/a self.append(False)
1100n/a dispatch[NEWFALSE[0]] = load_false
1101n/a
1102n/a def load_true(self):
1103n/a self.append(True)
1104n/a dispatch[NEWTRUE[0]] = load_true
1105n/a
1106n/a def load_int(self):
1107n/a data = self.readline()
1108n/a if data == FALSE[1:]:
1109n/a val = False
1110n/a elif data == TRUE[1:]:
1111n/a val = True
1112n/a else:
1113n/a val = int(data, 0)
1114n/a self.append(val)
1115n/a dispatch[INT[0]] = load_int
1116n/a
1117n/a def load_binint(self):
1118n/a self.append(unpack('<i', self.read(4))[0])
1119n/a dispatch[BININT[0]] = load_binint
1120n/a
1121n/a def load_binint1(self):
1122n/a self.append(self.read(1)[0])
1123n/a dispatch[BININT1[0]] = load_binint1
1124n/a
1125n/a def load_binint2(self):
1126n/a self.append(unpack('<H', self.read(2))[0])
1127n/a dispatch[BININT2[0]] = load_binint2
1128n/a
1129n/a def load_long(self):
1130n/a val = self.readline()[:-1]
1131n/a if val and val[-1] == b'L'[0]:
1132n/a val = val[:-1]
1133n/a self.append(int(val, 0))
1134n/a dispatch[LONG[0]] = load_long
1135n/a
1136n/a def load_long1(self):
1137n/a n = self.read(1)[0]
1138n/a data = self.read(n)
1139n/a self.append(decode_long(data))
1140n/a dispatch[LONG1[0]] = load_long1
1141n/a
1142n/a def load_long4(self):
1143n/a n, = unpack('<i', self.read(4))
1144n/a if n < 0:
1145n/a # Corrupt or hostile pickle -- we never write one like this
1146n/a raise UnpicklingError("LONG pickle has negative byte count")
1147n/a data = self.read(n)
1148n/a self.append(decode_long(data))
1149n/a dispatch[LONG4[0]] = load_long4
1150n/a
1151n/a def load_float(self):
1152n/a self.append(float(self.readline()[:-1]))
1153n/a dispatch[FLOAT[0]] = load_float
1154n/a
1155n/a def load_binfloat(self):
1156n/a self.append(unpack('>d', self.read(8))[0])
1157n/a dispatch[BINFLOAT[0]] = load_binfloat
1158n/a
1159n/a def _decode_string(self, value):
1160n/a # Used to allow strings from Python 2 to be decoded either as
1161n/a # bytes or Unicode strings. This should be used only with the
1162n/a # STRING, BINSTRING and SHORT_BINSTRING opcodes.
1163n/a if self.encoding == "bytes":
1164n/a return value
1165n/a else:
1166n/a return value.decode(self.encoding, self.errors)
1167n/a
1168n/a def load_string(self):
1169n/a data = self.readline()[:-1]
1170n/a # Strip outermost quotes
1171n/a if len(data) >= 2 and data[0] == data[-1] and data[0] in b'"\'':
1172n/a data = data[1:-1]
1173n/a else:
1174n/a raise UnpicklingError("the STRING opcode argument must be quoted")
1175n/a self.append(self._decode_string(codecs.escape_decode(data)[0]))
1176n/a dispatch[STRING[0]] = load_string
1177n/a
1178n/a def load_binstring(self):
1179n/a # Deprecated BINSTRING uses signed 32-bit length
1180n/a len, = unpack('<i', self.read(4))
1181n/a if len < 0:
1182n/a raise UnpicklingError("BINSTRING pickle has negative byte count")
1183n/a data = self.read(len)
1184n/a self.append(self._decode_string(data))
1185n/a dispatch[BINSTRING[0]] = load_binstring
1186n/a
1187n/a def load_binbytes(self):
1188n/a len, = unpack('<I', self.read(4))
1189n/a if len > maxsize:
1190n/a raise UnpicklingError("BINBYTES exceeds system's maximum size "
1191n/a "of %d bytes" % maxsize)
1192n/a self.append(self.read(len))
1193n/a dispatch[BINBYTES[0]] = load_binbytes
1194n/a
1195n/a def load_unicode(self):
1196n/a self.append(str(self.readline()[:-1], 'raw-unicode-escape'))
1197n/a dispatch[UNICODE[0]] = load_unicode
1198n/a
1199n/a def load_binunicode(self):
1200n/a len, = unpack('<I', self.read(4))
1201n/a if len > maxsize:
1202n/a raise UnpicklingError("BINUNICODE exceeds system's maximum size "
1203n/a "of %d bytes" % maxsize)
1204n/a self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
1205n/a dispatch[BINUNICODE[0]] = load_binunicode
1206n/a
1207n/a def load_binunicode8(self):
1208n/a len, = unpack('<Q', self.read(8))
1209n/a if len > maxsize:
1210n/a raise UnpicklingError("BINUNICODE8 exceeds system's maximum size "
1211n/a "of %d bytes" % maxsize)
1212n/a self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
1213n/a dispatch[BINUNICODE8[0]] = load_binunicode8
1214n/a
1215n/a def load_binbytes8(self):
1216n/a len, = unpack('<Q', self.read(8))
1217n/a if len > maxsize:
1218n/a raise UnpicklingError("BINBYTES8 exceeds system's maximum size "
1219n/a "of %d bytes" % maxsize)
1220n/a self.append(self.read(len))
1221n/a dispatch[BINBYTES8[0]] = load_binbytes8
1222n/a
1223n/a def load_short_binstring(self):
1224n/a len = self.read(1)[0]
1225n/a data = self.read(len)
1226n/a self.append(self._decode_string(data))
1227n/a dispatch[SHORT_BINSTRING[0]] = load_short_binstring
1228n/a
1229n/a def load_short_binbytes(self):
1230n/a len = self.read(1)[0]
1231n/a self.append(self.read(len))
1232n/a dispatch[SHORT_BINBYTES[0]] = load_short_binbytes
1233n/a
1234n/a def load_short_binunicode(self):
1235n/a len = self.read(1)[0]
1236n/a self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
1237n/a dispatch[SHORT_BINUNICODE[0]] = load_short_binunicode
1238n/a
1239n/a def load_tuple(self):
1240n/a items = self.pop_mark()
1241n/a self.append(tuple(items))
1242n/a dispatch[TUPLE[0]] = load_tuple
1243n/a
1244n/a def load_empty_tuple(self):
1245n/a self.append(())
1246n/a dispatch[EMPTY_TUPLE[0]] = load_empty_tuple
1247n/a
1248n/a def load_tuple1(self):
1249n/a self.stack[-1] = (self.stack[-1],)
1250n/a dispatch[TUPLE1[0]] = load_tuple1
1251n/a
1252n/a def load_tuple2(self):
1253n/a self.stack[-2:] = [(self.stack[-2], self.stack[-1])]
1254n/a dispatch[TUPLE2[0]] = load_tuple2
1255n/a
1256n/a def load_tuple3(self):
1257n/a self.stack[-3:] = [(self.stack[-3], self.stack[-2], self.stack[-1])]
1258n/a dispatch[TUPLE3[0]] = load_tuple3
1259n/a
1260n/a def load_empty_list(self):
1261n/a self.append([])
1262n/a dispatch[EMPTY_LIST[0]] = load_empty_list
1263n/a
1264n/a def load_empty_dictionary(self):
1265n/a self.append({})
1266n/a dispatch[EMPTY_DICT[0]] = load_empty_dictionary
1267n/a
1268n/a def load_empty_set(self):
1269n/a self.append(set())
1270n/a dispatch[EMPTY_SET[0]] = load_empty_set
1271n/a
1272n/a def load_frozenset(self):
1273n/a items = self.pop_mark()
1274n/a self.append(frozenset(items))
1275n/a dispatch[FROZENSET[0]] = load_frozenset
1276n/a
1277n/a def load_list(self):
1278n/a items = self.pop_mark()
1279n/a self.append(items)
1280n/a dispatch[LIST[0]] = load_list
1281n/a
1282n/a def load_dict(self):
1283n/a items = self.pop_mark()
1284n/a d = {items[i]: items[i+1]
1285n/a for i in range(0, len(items), 2)}
1286n/a self.append(d)
1287n/a dispatch[DICT[0]] = load_dict
1288n/a
1289n/a # INST and OBJ differ only in how they get a class object. It's not
1290n/a # only sensible to do the rest in a common routine, the two routines
1291n/a # previously diverged and grew different bugs.
1292n/a # klass is the class to instantiate, and k points to the topmost mark
1293n/a # object, following which are the arguments for klass.__init__.
1294n/a def _instantiate(self, klass, args):
1295n/a if (args or not isinstance(klass, type) or
1296n/a hasattr(klass, "__getinitargs__")):
1297n/a try:
1298n/a value = klass(*args)
1299n/a except TypeError as err:
1300n/a raise TypeError("in constructor for %s: %s" %
1301n/a (klass.__name__, str(err)), sys.exc_info()[2])
1302n/a else:
1303n/a value = klass.__new__(klass)
1304n/a self.append(value)
1305n/a
1306n/a def load_inst(self):
1307n/a module = self.readline()[:-1].decode("ascii")
1308n/a name = self.readline()[:-1].decode("ascii")
1309n/a klass = self.find_class(module, name)
1310n/a self._instantiate(klass, self.pop_mark())
1311n/a dispatch[INST[0]] = load_inst
1312n/a
1313n/a def load_obj(self):
1314n/a # Stack is ... markobject classobject arg1 arg2 ...
1315n/a args = self.pop_mark()
1316n/a cls = args.pop(0)
1317n/a self._instantiate(cls, args)
1318n/a dispatch[OBJ[0]] = load_obj
1319n/a
1320n/a def load_newobj(self):
1321n/a args = self.stack.pop()
1322n/a cls = self.stack.pop()
1323n/a obj = cls.__new__(cls, *args)
1324n/a self.append(obj)
1325n/a dispatch[NEWOBJ[0]] = load_newobj
1326n/a
1327n/a def load_newobj_ex(self):
1328n/a kwargs = self.stack.pop()
1329n/a args = self.stack.pop()
1330n/a cls = self.stack.pop()
1331n/a obj = cls.__new__(cls, *args, **kwargs)
1332n/a self.append(obj)
1333n/a dispatch[NEWOBJ_EX[0]] = load_newobj_ex
1334n/a
1335n/a def load_global(self):
1336n/a module = self.readline()[:-1].decode("utf-8")
1337n/a name = self.readline()[:-1].decode("utf-8")
1338n/a klass = self.find_class(module, name)
1339n/a self.append(klass)
1340n/a dispatch[GLOBAL[0]] = load_global
1341n/a
1342n/a def load_stack_global(self):
1343n/a name = self.stack.pop()
1344n/a module = self.stack.pop()
1345n/a if type(name) is not str or type(module) is not str:
1346n/a raise UnpicklingError("STACK_GLOBAL requires str")
1347n/a self.append(self.find_class(module, name))
1348n/a dispatch[STACK_GLOBAL[0]] = load_stack_global
1349n/a
1350n/a def load_ext1(self):
1351n/a code = self.read(1)[0]
1352n/a self.get_extension(code)
1353n/a dispatch[EXT1[0]] = load_ext1
1354n/a
1355n/a def load_ext2(self):
1356n/a code, = unpack('<H', self.read(2))
1357n/a self.get_extension(code)
1358n/a dispatch[EXT2[0]] = load_ext2
1359n/a
1360n/a def load_ext4(self):
1361n/a code, = unpack('<i', self.read(4))
1362n/a self.get_extension(code)
1363n/a dispatch[EXT4[0]] = load_ext4
1364n/a
1365n/a def get_extension(self, code):
1366n/a nil = []
1367n/a obj = _extension_cache.get(code, nil)
1368n/a if obj is not nil:
1369n/a self.append(obj)
1370n/a return
1371n/a key = _inverted_registry.get(code)
1372n/a if not key:
1373n/a if code <= 0: # note that 0 is forbidden
1374n/a # Corrupt or hostile pickle.
1375n/a raise UnpicklingError("EXT specifies code <= 0")
1376n/a raise ValueError("unregistered extension code %d" % code)
1377n/a obj = self.find_class(*key)
1378n/a _extension_cache[code] = obj
1379n/a self.append(obj)
1380n/a
1381n/a def find_class(self, module, name):
1382n/a # Subclasses may override this.
1383n/a if self.proto < 3 and self.fix_imports:
1384n/a if (module, name) in _compat_pickle.NAME_MAPPING:
1385n/a module, name = _compat_pickle.NAME_MAPPING[(module, name)]
1386n/a elif module in _compat_pickle.IMPORT_MAPPING:
1387n/a module = _compat_pickle.IMPORT_MAPPING[module]
1388n/a __import__(module, level=0)
1389n/a if self.proto >= 4:
1390n/a return _getattribute(sys.modules[module], name)[0]
1391n/a else:
1392n/a return getattr(sys.modules[module], name)
1393n/a
1394n/a def load_reduce(self):
1395n/a stack = self.stack
1396n/a args = stack.pop()
1397n/a func = stack[-1]
1398n/a stack[-1] = func(*args)
1399n/a dispatch[REDUCE[0]] = load_reduce
1400n/a
1401n/a def load_pop(self):
1402n/a if self.stack:
1403n/a del self.stack[-1]
1404n/a else:
1405n/a self.pop_mark()
1406n/a dispatch[POP[0]] = load_pop
1407n/a
1408n/a def load_pop_mark(self):
1409n/a self.pop_mark()
1410n/a dispatch[POP_MARK[0]] = load_pop_mark
1411n/a
1412n/a def load_dup(self):
1413n/a self.append(self.stack[-1])
1414n/a dispatch[DUP[0]] = load_dup
1415n/a
1416n/a def load_get(self):
1417n/a i = int(self.readline()[:-1])
1418n/a self.append(self.memo[i])
1419n/a dispatch[GET[0]] = load_get
1420n/a
1421n/a def load_binget(self):
1422n/a i = self.read(1)[0]
1423n/a self.append(self.memo[i])
1424n/a dispatch[BINGET[0]] = load_binget
1425n/a
1426n/a def load_long_binget(self):
1427n/a i, = unpack('<I', self.read(4))
1428n/a self.append(self.memo[i])
1429n/a dispatch[LONG_BINGET[0]] = load_long_binget
1430n/a
1431n/a def load_put(self):
1432n/a i = int(self.readline()[:-1])
1433n/a if i < 0:
1434n/a raise ValueError("negative PUT argument")
1435n/a self.memo[i] = self.stack[-1]
1436n/a dispatch[PUT[0]] = load_put
1437n/a
1438n/a def load_binput(self):
1439n/a i = self.read(1)[0]
1440n/a if i < 0:
1441n/a raise ValueError("negative BINPUT argument")
1442n/a self.memo[i] = self.stack[-1]
1443n/a dispatch[BINPUT[0]] = load_binput
1444n/a
1445n/a def load_long_binput(self):
1446n/a i, = unpack('<I', self.read(4))
1447n/a if i > maxsize:
1448n/a raise ValueError("negative LONG_BINPUT argument")
1449n/a self.memo[i] = self.stack[-1]
1450n/a dispatch[LONG_BINPUT[0]] = load_long_binput
1451n/a
1452n/a def load_memoize(self):
1453n/a memo = self.memo
1454n/a memo[len(memo)] = self.stack[-1]
1455n/a dispatch[MEMOIZE[0]] = load_memoize
1456n/a
1457n/a def load_append(self):
1458n/a stack = self.stack
1459n/a value = stack.pop()
1460n/a list = stack[-1]
1461n/a list.append(value)
1462n/a dispatch[APPEND[0]] = load_append
1463n/a
1464n/a def load_appends(self):
1465n/a items = self.pop_mark()
1466n/a list_obj = self.stack[-1]
1467n/a try:
1468n/a extend = list_obj.extend
1469n/a except AttributeError:
1470n/a pass
1471n/a else:
1472n/a extend(items)
1473n/a return
1474n/a # Even if the PEP 307 requires extend() and append() methods,
1475n/a # fall back on append() if the object has no extend() method
1476n/a # for backward compatibility.
1477n/a append = list_obj.append
1478n/a for item in items:
1479n/a append(item)
1480n/a dispatch[APPENDS[0]] = load_appends
1481n/a
1482n/a def load_setitem(self):
1483n/a stack = self.stack
1484n/a value = stack.pop()
1485n/a key = stack.pop()
1486n/a dict = stack[-1]
1487n/a dict[key] = value
1488n/a dispatch[SETITEM[0]] = load_setitem
1489n/a
1490n/a def load_setitems(self):
1491n/a items = self.pop_mark()
1492n/a dict = self.stack[-1]
1493n/a for i in range(0, len(items), 2):
1494n/a dict[items[i]] = items[i + 1]
1495n/a dispatch[SETITEMS[0]] = load_setitems
1496n/a
1497n/a def load_additems(self):
1498n/a items = self.pop_mark()
1499n/a set_obj = self.stack[-1]
1500n/a if isinstance(set_obj, set):
1501n/a set_obj.update(items)
1502n/a else:
1503n/a add = set_obj.add
1504n/a for item in items:
1505n/a add(item)
1506n/a dispatch[ADDITEMS[0]] = load_additems
1507n/a
1508n/a def load_build(self):
1509n/a stack = self.stack
1510n/a state = stack.pop()
1511n/a inst = stack[-1]
1512n/a setstate = getattr(inst, "__setstate__", None)
1513n/a if setstate is not None:
1514n/a setstate(state)
1515n/a return
1516n/a slotstate = None
1517n/a if isinstance(state, tuple) and len(state) == 2:
1518n/a state, slotstate = state
1519n/a if state:
1520n/a inst_dict = inst.__dict__
1521n/a intern = sys.intern
1522n/a for k, v in state.items():
1523n/a if type(k) is str:
1524n/a inst_dict[intern(k)] = v
1525n/a else:
1526n/a inst_dict[k] = v
1527n/a if slotstate:
1528n/a for k, v in slotstate.items():
1529n/a setattr(inst, k, v)
1530n/a dispatch[BUILD[0]] = load_build
1531n/a
1532n/a def load_mark(self):
1533n/a self.metastack.append(self.stack)
1534n/a self.stack = []
1535n/a self.append = self.stack.append
1536n/a dispatch[MARK[0]] = load_mark
1537n/a
1538n/a def load_stop(self):
1539n/a value = self.stack.pop()
1540n/a raise _Stop(value)
1541n/a dispatch[STOP[0]] = load_stop
1542n/a
1543n/a
1544n/a# Shorthands
1545n/a
1546n/adef _dump(obj, file, protocol=None, *, fix_imports=True):
1547n/a _Pickler(file, protocol, fix_imports=fix_imports).dump(obj)
1548n/a
1549n/adef _dumps(obj, protocol=None, *, fix_imports=True):
1550n/a f = io.BytesIO()
1551n/a _Pickler(f, protocol, fix_imports=fix_imports).dump(obj)
1552n/a res = f.getvalue()
1553n/a assert isinstance(res, bytes_types)
1554n/a return res
1555n/a
1556n/adef _load(file, *, fix_imports=True, encoding="ASCII", errors="strict"):
1557n/a return _Unpickler(file, fix_imports=fix_imports,
1558n/a encoding=encoding, errors=errors).load()
1559n/a
1560n/adef _loads(s, *, fix_imports=True, encoding="ASCII", errors="strict"):
1561n/a if isinstance(s, str):
1562n/a raise TypeError("Can't load pickle from unicode string")
1563n/a file = io.BytesIO(s)
1564n/a return _Unpickler(file, fix_imports=fix_imports,
1565n/a encoding=encoding, errors=errors).load()
1566n/a
1567n/a# Use the faster _pickle if possible
1568n/atry:
1569n/a from _pickle import (
1570n/a PickleError,
1571n/a PicklingError,
1572n/a UnpicklingError,
1573n/a Pickler,
1574n/a Unpickler,
1575n/a dump,
1576n/a dumps,
1577n/a load,
1578n/a loads
1579n/a )
1580n/aexcept ImportError:
1581n/a Pickler, Unpickler = _Pickler, _Unpickler
1582n/a dump, dumps, load, loads = _dump, _dumps, _load, _loads
1583n/a
1584n/a# Doctest
1585n/adef _test():
1586n/a import doctest
1587n/a return doctest.testmod()
1588n/a
1589n/aif __name__ == "__main__":
1590n/a import argparse
1591n/a parser = argparse.ArgumentParser(
1592n/a description='display contents of the pickle files')
1593n/a parser.add_argument(
1594n/a 'pickle_file', type=argparse.FileType('br'),
1595n/a nargs='*', help='the pickle file')
1596n/a parser.add_argument(
1597n/a '-t', '--test', action='store_true',
1598n/a help='run self-test suite')
1599n/a parser.add_argument(
1600n/a '-v', action='store_true',
1601n/a help='run verbosely; only affects self-test run')
1602n/a args = parser.parse_args()
1603n/a if args.test:
1604n/a _test()
1605n/a else:
1606n/a if not args.pickle_file:
1607n/a parser.print_help()
1608n/a else:
1609n/a import pprint
1610n/a for f in args.pickle_file:
1611n/a obj = load(f)
1612n/a pprint.pprint(obj)