ยปCore Development>Code coverage>Modules/_pickle.c

Python code coverage for Modules/_pickle.c

#countcontent
1n/a#include "Python.h"
2n/a#include "structmember.h"
3n/a
4n/aPyDoc_STRVAR(pickle_module_doc,
5n/a"Optimized C implementation for the Python pickle module.");
6n/a
7n/a/*[clinic input]
8n/amodule _pickle
9n/aclass _pickle.Pickler "PicklerObject *" "&Pickler_Type"
10n/aclass _pickle.PicklerMemoProxy "PicklerMemoProxyObject *" "&PicklerMemoProxyType"
11n/aclass _pickle.Unpickler "UnpicklerObject *" "&Unpickler_Type"
12n/aclass _pickle.UnpicklerMemoProxy "UnpicklerMemoProxyObject *" "&UnpicklerMemoProxyType"
13n/a[clinic start generated code]*/
14n/a/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4b3e113468a58e6c]*/
15n/a
16n/a/* Bump this when new opcodes are added to the pickle protocol. */
17n/aenum {
18n/a HIGHEST_PROTOCOL = 4,
19n/a DEFAULT_PROTOCOL = 3
20n/a};
21n/a
22n/a/* Pickle opcodes. These must be kept updated with pickle.py.
23n/a Extensive docs are in pickletools.py. */
24n/aenum opcode {
25n/a MARK = '(',
26n/a STOP = '.',
27n/a POP = '0',
28n/a POP_MARK = '1',
29n/a DUP = '2',
30n/a FLOAT = 'F',
31n/a INT = 'I',
32n/a BININT = 'J',
33n/a BININT1 = 'K',
34n/a LONG = 'L',
35n/a BININT2 = 'M',
36n/a NONE = 'N',
37n/a PERSID = 'P',
38n/a BINPERSID = 'Q',
39n/a REDUCE = 'R',
40n/a STRING = 'S',
41n/a BINSTRING = 'T',
42n/a SHORT_BINSTRING = 'U',
43n/a UNICODE = 'V',
44n/a BINUNICODE = 'X',
45n/a APPEND = 'a',
46n/a BUILD = 'b',
47n/a GLOBAL = 'c',
48n/a DICT = 'd',
49n/a EMPTY_DICT = '}',
50n/a APPENDS = 'e',
51n/a GET = 'g',
52n/a BINGET = 'h',
53n/a INST = 'i',
54n/a LONG_BINGET = 'j',
55n/a LIST = 'l',
56n/a EMPTY_LIST = ']',
57n/a OBJ = 'o',
58n/a PUT = 'p',
59n/a BINPUT = 'q',
60n/a LONG_BINPUT = 'r',
61n/a SETITEM = 's',
62n/a TUPLE = 't',
63n/a EMPTY_TUPLE = ')',
64n/a SETITEMS = 'u',
65n/a BINFLOAT = 'G',
66n/a
67n/a /* Protocol 2. */
68n/a PROTO = '\x80',
69n/a NEWOBJ = '\x81',
70n/a EXT1 = '\x82',
71n/a EXT2 = '\x83',
72n/a EXT4 = '\x84',
73n/a TUPLE1 = '\x85',
74n/a TUPLE2 = '\x86',
75n/a TUPLE3 = '\x87',
76n/a NEWTRUE = '\x88',
77n/a NEWFALSE = '\x89',
78n/a LONG1 = '\x8a',
79n/a LONG4 = '\x8b',
80n/a
81n/a /* Protocol 3 (Python 3.x) */
82n/a BINBYTES = 'B',
83n/a SHORT_BINBYTES = 'C',
84n/a
85n/a /* Protocol 4 */
86n/a SHORT_BINUNICODE = '\x8c',
87n/a BINUNICODE8 = '\x8d',
88n/a BINBYTES8 = '\x8e',
89n/a EMPTY_SET = '\x8f',
90n/a ADDITEMS = '\x90',
91n/a FROZENSET = '\x91',
92n/a NEWOBJ_EX = '\x92',
93n/a STACK_GLOBAL = '\x93',
94n/a MEMOIZE = '\x94',
95n/a FRAME = '\x95'
96n/a};
97n/a
98n/aenum {
99n/a /* Keep in synch with pickle.Pickler._BATCHSIZE. This is how many elements
100n/a batch_list/dict() pumps out before doing APPENDS/SETITEMS. Nothing will
101n/a break if this gets out of synch with pickle.py, but it's unclear that would
102n/a help anything either. */
103n/a BATCHSIZE = 1000,
104n/a
105n/a /* Nesting limit until Pickler, when running in "fast mode", starts
106n/a checking for self-referential data-structures. */
107n/a FAST_NESTING_LIMIT = 50,
108n/a
109n/a /* Initial size of the write buffer of Pickler. */
110n/a WRITE_BUF_SIZE = 4096,
111n/a
112n/a /* Prefetch size when unpickling (disabled on unpeekable streams) */
113n/a PREFETCH = 8192 * 16,
114n/a
115n/a FRAME_SIZE_TARGET = 64 * 1024,
116n/a
117n/a FRAME_HEADER_SIZE = 9
118n/a};
119n/a
120n/a/*************************************************************************/
121n/a
122n/a/* State of the pickle module, per PEP 3121. */
123n/atypedef struct {
124n/a /* Exception classes for pickle. */
125n/a PyObject *PickleError;
126n/a PyObject *PicklingError;
127n/a PyObject *UnpicklingError;
128n/a
129n/a /* copyreg.dispatch_table, {type_object: pickling_function} */
130n/a PyObject *dispatch_table;
131n/a
132n/a /* For the extension opcodes EXT1, EXT2 and EXT4. */
133n/a
134n/a /* copyreg._extension_registry, {(module_name, function_name): code} */
135n/a PyObject *extension_registry;
136n/a /* copyreg._extension_cache, {code: object} */
137n/a PyObject *extension_cache;
138n/a /* copyreg._inverted_registry, {code: (module_name, function_name)} */
139n/a PyObject *inverted_registry;
140n/a
141n/a /* Import mappings for compatibility with Python 2.x */
142n/a
143n/a /* _compat_pickle.NAME_MAPPING,
144n/a {(oldmodule, oldname): (newmodule, newname)} */
145n/a PyObject *name_mapping_2to3;
146n/a /* _compat_pickle.IMPORT_MAPPING, {oldmodule: newmodule} */
147n/a PyObject *import_mapping_2to3;
148n/a /* Same, but with REVERSE_NAME_MAPPING / REVERSE_IMPORT_MAPPING */
149n/a PyObject *name_mapping_3to2;
150n/a PyObject *import_mapping_3to2;
151n/a
152n/a /* codecs.encode, used for saving bytes in older protocols */
153n/a PyObject *codecs_encode;
154n/a /* builtins.getattr, used for saving nested names with protocol < 4 */
155n/a PyObject *getattr;
156n/a /* functools.partial, used for implementing __newobj_ex__ with protocols
157n/a 2 and 3 */
158n/a PyObject *partial;
159n/a} PickleState;
160n/a
161n/a/* Forward declaration of the _pickle module definition. */
162n/astatic struct PyModuleDef _picklemodule;
163n/a
164n/a/* Given a module object, get its per-module state. */
165n/astatic PickleState *
166n/a_Pickle_GetState(PyObject *module)
167n/a{
168n/a return (PickleState *)PyModule_GetState(module);
169n/a}
170n/a
171n/a/* Find the module instance imported in the currently running sub-interpreter
172n/a and get its state. */
173n/astatic PickleState *
174n/a_Pickle_GetGlobalState(void)
175n/a{
176n/a return _Pickle_GetState(PyState_FindModule(&_picklemodule));
177n/a}
178n/a
179n/a/* Clear the given pickle module state. */
180n/astatic void
181n/a_Pickle_ClearState(PickleState *st)
182n/a{
183n/a Py_CLEAR(st->PickleError);
184n/a Py_CLEAR(st->PicklingError);
185n/a Py_CLEAR(st->UnpicklingError);
186n/a Py_CLEAR(st->dispatch_table);
187n/a Py_CLEAR(st->extension_registry);
188n/a Py_CLEAR(st->extension_cache);
189n/a Py_CLEAR(st->inverted_registry);
190n/a Py_CLEAR(st->name_mapping_2to3);
191n/a Py_CLEAR(st->import_mapping_2to3);
192n/a Py_CLEAR(st->name_mapping_3to2);
193n/a Py_CLEAR(st->import_mapping_3to2);
194n/a Py_CLEAR(st->codecs_encode);
195n/a Py_CLEAR(st->getattr);
196n/a Py_CLEAR(st->partial);
197n/a}
198n/a
199n/a/* Initialize the given pickle module state. */
200n/astatic int
201n/a_Pickle_InitState(PickleState *st)
202n/a{
203n/a PyObject *builtins;
204n/a PyObject *copyreg = NULL;
205n/a PyObject *compat_pickle = NULL;
206n/a PyObject *codecs = NULL;
207n/a PyObject *functools = NULL;
208n/a
209n/a builtins = PyEval_GetBuiltins();
210n/a if (builtins == NULL)
211n/a goto error;
212n/a st->getattr = PyDict_GetItemString(builtins, "getattr");
213n/a if (st->getattr == NULL)
214n/a goto error;
215n/a Py_INCREF(st->getattr);
216n/a
217n/a copyreg = PyImport_ImportModule("copyreg");
218n/a if (!copyreg)
219n/a goto error;
220n/a st->dispatch_table = PyObject_GetAttrString(copyreg, "dispatch_table");
221n/a if (!st->dispatch_table)
222n/a goto error;
223n/a if (!PyDict_CheckExact(st->dispatch_table)) {
224n/a PyErr_Format(PyExc_RuntimeError,
225n/a "copyreg.dispatch_table should be a dict, not %.200s",
226n/a Py_TYPE(st->dispatch_table)->tp_name);
227n/a goto error;
228n/a }
229n/a st->extension_registry = \
230n/a PyObject_GetAttrString(copyreg, "_extension_registry");
231n/a if (!st->extension_registry)
232n/a goto error;
233n/a if (!PyDict_CheckExact(st->extension_registry)) {
234n/a PyErr_Format(PyExc_RuntimeError,
235n/a "copyreg._extension_registry should be a dict, "
236n/a "not %.200s", Py_TYPE(st->extension_registry)->tp_name);
237n/a goto error;
238n/a }
239n/a st->inverted_registry = \
240n/a PyObject_GetAttrString(copyreg, "_inverted_registry");
241n/a if (!st->inverted_registry)
242n/a goto error;
243n/a if (!PyDict_CheckExact(st->inverted_registry)) {
244n/a PyErr_Format(PyExc_RuntimeError,
245n/a "copyreg._inverted_registry should be a dict, "
246n/a "not %.200s", Py_TYPE(st->inverted_registry)->tp_name);
247n/a goto error;
248n/a }
249n/a st->extension_cache = PyObject_GetAttrString(copyreg, "_extension_cache");
250n/a if (!st->extension_cache)
251n/a goto error;
252n/a if (!PyDict_CheckExact(st->extension_cache)) {
253n/a PyErr_Format(PyExc_RuntimeError,
254n/a "copyreg._extension_cache should be a dict, "
255n/a "not %.200s", Py_TYPE(st->extension_cache)->tp_name);
256n/a goto error;
257n/a }
258n/a Py_CLEAR(copyreg);
259n/a
260n/a /* Load the 2.x -> 3.x stdlib module mapping tables */
261n/a compat_pickle = PyImport_ImportModule("_compat_pickle");
262n/a if (!compat_pickle)
263n/a goto error;
264n/a st->name_mapping_2to3 = \
265n/a PyObject_GetAttrString(compat_pickle, "NAME_MAPPING");
266n/a if (!st->name_mapping_2to3)
267n/a goto error;
268n/a if (!PyDict_CheckExact(st->name_mapping_2to3)) {
269n/a PyErr_Format(PyExc_RuntimeError,
270n/a "_compat_pickle.NAME_MAPPING should be a dict, not %.200s",
271n/a Py_TYPE(st->name_mapping_2to3)->tp_name);
272n/a goto error;
273n/a }
274n/a st->import_mapping_2to3 = \
275n/a PyObject_GetAttrString(compat_pickle, "IMPORT_MAPPING");
276n/a if (!st->import_mapping_2to3)
277n/a goto error;
278n/a if (!PyDict_CheckExact(st->import_mapping_2to3)) {
279n/a PyErr_Format(PyExc_RuntimeError,
280n/a "_compat_pickle.IMPORT_MAPPING should be a dict, "
281n/a "not %.200s", Py_TYPE(st->import_mapping_2to3)->tp_name);
282n/a goto error;
283n/a }
284n/a /* ... and the 3.x -> 2.x mapping tables */
285n/a st->name_mapping_3to2 = \
286n/a PyObject_GetAttrString(compat_pickle, "REVERSE_NAME_MAPPING");
287n/a if (!st->name_mapping_3to2)
288n/a goto error;
289n/a if (!PyDict_CheckExact(st->name_mapping_3to2)) {
290n/a PyErr_Format(PyExc_RuntimeError,
291n/a "_compat_pickle.REVERSE_NAME_MAPPING should be a dict, "
292n/a "not %.200s", Py_TYPE(st->name_mapping_3to2)->tp_name);
293n/a goto error;
294n/a }
295n/a st->import_mapping_3to2 = \
296n/a PyObject_GetAttrString(compat_pickle, "REVERSE_IMPORT_MAPPING");
297n/a if (!st->import_mapping_3to2)
298n/a goto error;
299n/a if (!PyDict_CheckExact(st->import_mapping_3to2)) {
300n/a PyErr_Format(PyExc_RuntimeError,
301n/a "_compat_pickle.REVERSE_IMPORT_MAPPING should be a dict, "
302n/a "not %.200s", Py_TYPE(st->import_mapping_3to2)->tp_name);
303n/a goto error;
304n/a }
305n/a Py_CLEAR(compat_pickle);
306n/a
307n/a codecs = PyImport_ImportModule("codecs");
308n/a if (codecs == NULL)
309n/a goto error;
310n/a st->codecs_encode = PyObject_GetAttrString(codecs, "encode");
311n/a if (st->codecs_encode == NULL) {
312n/a goto error;
313n/a }
314n/a if (!PyCallable_Check(st->codecs_encode)) {
315n/a PyErr_Format(PyExc_RuntimeError,
316n/a "codecs.encode should be a callable, not %.200s",
317n/a Py_TYPE(st->codecs_encode)->tp_name);
318n/a goto error;
319n/a }
320n/a Py_CLEAR(codecs);
321n/a
322n/a functools = PyImport_ImportModule("functools");
323n/a if (!functools)
324n/a goto error;
325n/a st->partial = PyObject_GetAttrString(functools, "partial");
326n/a if (!st->partial)
327n/a goto error;
328n/a Py_CLEAR(functools);
329n/a
330n/a return 0;
331n/a
332n/a error:
333n/a Py_CLEAR(copyreg);
334n/a Py_CLEAR(compat_pickle);
335n/a Py_CLEAR(codecs);
336n/a Py_CLEAR(functools);
337n/a _Pickle_ClearState(st);
338n/a return -1;
339n/a}
340n/a
341n/a/* Helper for calling a function with a single argument quickly.
342n/a
343n/a This function steals the reference of the given argument. */
344n/astatic PyObject *
345n/a_Pickle_FastCall(PyObject *func, PyObject *obj)
346n/a{
347n/a PyObject *result;
348n/a
349n/a result = PyObject_CallFunctionObjArgs(func, obj, NULL);
350n/a Py_DECREF(obj);
351n/a return result;
352n/a}
353n/a
354n/a/*************************************************************************/
355n/a
356n/a/* Internal data type used as the unpickling stack. */
357n/atypedef struct {
358n/a PyObject_VAR_HEAD
359n/a PyObject **data;
360n/a int mark_set; /* is MARK set? */
361n/a Py_ssize_t fence; /* position of top MARK or 0 */
362n/a Py_ssize_t allocated; /* number of slots in data allocated */
363n/a} Pdata;
364n/a
365n/astatic void
366n/aPdata_dealloc(Pdata *self)
367n/a{
368n/a Py_ssize_t i = Py_SIZE(self);
369n/a while (--i >= 0) {
370n/a Py_DECREF(self->data[i]);
371n/a }
372n/a PyMem_FREE(self->data);
373n/a PyObject_Del(self);
374n/a}
375n/a
376n/astatic PyTypeObject Pdata_Type = {
377n/a PyVarObject_HEAD_INIT(NULL, 0)
378n/a "_pickle.Pdata", /*tp_name*/
379n/a sizeof(Pdata), /*tp_basicsize*/
380n/a sizeof(PyObject *), /*tp_itemsize*/
381n/a (destructor)Pdata_dealloc, /*tp_dealloc*/
382n/a};
383n/a
384n/astatic PyObject *
385n/aPdata_New(void)
386n/a{
387n/a Pdata *self;
388n/a
389n/a if (!(self = PyObject_New(Pdata, &Pdata_Type)))
390n/a return NULL;
391n/a Py_SIZE(self) = 0;
392n/a self->mark_set = 0;
393n/a self->fence = 0;
394n/a self->allocated = 8;
395n/a self->data = PyMem_MALLOC(self->allocated * sizeof(PyObject *));
396n/a if (self->data)
397n/a return (PyObject *)self;
398n/a Py_DECREF(self);
399n/a return PyErr_NoMemory();
400n/a}
401n/a
402n/a
403n/a/* Retain only the initial clearto items. If clearto >= the current
404n/a * number of items, this is a (non-erroneous) NOP.
405n/a */
406n/astatic int
407n/aPdata_clear(Pdata *self, Py_ssize_t clearto)
408n/a{
409n/a Py_ssize_t i = Py_SIZE(self);
410n/a
411n/a assert(clearto >= self->fence);
412n/a if (clearto >= i)
413n/a return 0;
414n/a
415n/a while (--i >= clearto) {
416n/a Py_CLEAR(self->data[i]);
417n/a }
418n/a Py_SIZE(self) = clearto;
419n/a return 0;
420n/a}
421n/a
422n/astatic int
423n/aPdata_grow(Pdata *self)
424n/a{
425n/a PyObject **data = self->data;
426n/a size_t allocated = (size_t)self->allocated;
427n/a size_t new_allocated;
428n/a
429n/a new_allocated = (allocated >> 3) + 6;
430n/a /* check for integer overflow */
431n/a if (new_allocated > (size_t)PY_SSIZE_T_MAX - allocated)
432n/a goto nomemory;
433n/a new_allocated += allocated;
434n/a PyMem_RESIZE(data, PyObject *, new_allocated);
435n/a if (data == NULL)
436n/a goto nomemory;
437n/a
438n/a self->data = data;
439n/a self->allocated = (Py_ssize_t)new_allocated;
440n/a return 0;
441n/a
442n/a nomemory:
443n/a PyErr_NoMemory();
444n/a return -1;
445n/a}
446n/a
447n/astatic int
448n/aPdata_stack_underflow(Pdata *self)
449n/a{
450n/a PickleState *st = _Pickle_GetGlobalState();
451n/a PyErr_SetString(st->UnpicklingError,
452n/a self->mark_set ?
453n/a "unexpected MARK found" :
454n/a "unpickling stack underflow");
455n/a return -1;
456n/a}
457n/a
458n/a/* D is a Pdata*. Pop the topmost element and store it into V, which
459n/a * must be an lvalue holding PyObject*. On stack underflow, UnpicklingError
460n/a * is raised and V is set to NULL.
461n/a */
462n/astatic PyObject *
463n/aPdata_pop(Pdata *self)
464n/a{
465n/a if (Py_SIZE(self) <= self->fence) {
466n/a Pdata_stack_underflow(self);
467n/a return NULL;
468n/a }
469n/a return self->data[--Py_SIZE(self)];
470n/a}
471n/a#define PDATA_POP(D, V) do { (V) = Pdata_pop((D)); } while (0)
472n/a
473n/astatic int
474n/aPdata_push(Pdata *self, PyObject *obj)
475n/a{
476n/a if (Py_SIZE(self) == self->allocated && Pdata_grow(self) < 0) {
477n/a return -1;
478n/a }
479n/a self->data[Py_SIZE(self)++] = obj;
480n/a return 0;
481n/a}
482n/a
483n/a/* Push an object on stack, transferring its ownership to the stack. */
484n/a#define PDATA_PUSH(D, O, ER) do { \
485n/a if (Pdata_push((D), (O)) < 0) return (ER); } while(0)
486n/a
487n/a/* Push an object on stack, adding a new reference to the object. */
488n/a#define PDATA_APPEND(D, O, ER) do { \
489n/a Py_INCREF((O)); \
490n/a if (Pdata_push((D), (O)) < 0) return (ER); } while(0)
491n/a
492n/astatic PyObject *
493n/aPdata_poptuple(Pdata *self, Py_ssize_t start)
494n/a{
495n/a PyObject *tuple;
496n/a Py_ssize_t len, i, j;
497n/a
498n/a if (start < self->fence) {
499n/a Pdata_stack_underflow(self);
500n/a return NULL;
501n/a }
502n/a len = Py_SIZE(self) - start;
503n/a tuple = PyTuple_New(len);
504n/a if (tuple == NULL)
505n/a return NULL;
506n/a for (i = start, j = 0; j < len; i++, j++)
507n/a PyTuple_SET_ITEM(tuple, j, self->data[i]);
508n/a
509n/a Py_SIZE(self) = start;
510n/a return tuple;
511n/a}
512n/a
513n/astatic PyObject *
514n/aPdata_poplist(Pdata *self, Py_ssize_t start)
515n/a{
516n/a PyObject *list;
517n/a Py_ssize_t len, i, j;
518n/a
519n/a len = Py_SIZE(self) - start;
520n/a list = PyList_New(len);
521n/a if (list == NULL)
522n/a return NULL;
523n/a for (i = start, j = 0; j < len; i++, j++)
524n/a PyList_SET_ITEM(list, j, self->data[i]);
525n/a
526n/a Py_SIZE(self) = start;
527n/a return list;
528n/a}
529n/a
530n/atypedef struct {
531n/a PyObject *me_key;
532n/a Py_ssize_t me_value;
533n/a} PyMemoEntry;
534n/a
535n/atypedef struct {
536n/a Py_ssize_t mt_mask;
537n/a Py_ssize_t mt_used;
538n/a Py_ssize_t mt_allocated;
539n/a PyMemoEntry *mt_table;
540n/a} PyMemoTable;
541n/a
542n/atypedef struct PicklerObject {
543n/a PyObject_HEAD
544n/a PyMemoTable *memo; /* Memo table, keep track of the seen
545n/a objects to support self-referential objects
546n/a pickling. */
547n/a PyObject *pers_func; /* persistent_id() method, can be NULL */
548n/a PyObject *dispatch_table; /* private dispatch_table, can be NULL */
549n/a
550n/a PyObject *write; /* write() method of the output stream. */
551n/a PyObject *output_buffer; /* Write into a local bytearray buffer before
552n/a flushing to the stream. */
553n/a Py_ssize_t output_len; /* Length of output_buffer. */
554n/a Py_ssize_t max_output_len; /* Allocation size of output_buffer. */
555n/a int proto; /* Pickle protocol number, >= 0 */
556n/a int bin; /* Boolean, true if proto > 0 */
557n/a int framing; /* True when framing is enabled, proto >= 4 */
558n/a Py_ssize_t frame_start; /* Position in output_buffer where the
559n/a current frame begins. -1 if there
560n/a is no frame currently open. */
561n/a
562n/a Py_ssize_t buf_size; /* Size of the current buffered pickle data */
563n/a int fast; /* Enable fast mode if set to a true value.
564n/a The fast mode disable the usage of memo,
565n/a therefore speeding the pickling process by
566n/a not generating superfluous PUT opcodes. It
567n/a should not be used if with self-referential
568n/a objects. */
569n/a int fast_nesting;
570n/a int fix_imports; /* Indicate whether Pickler should fix
571n/a the name of globals for Python 2.x. */
572n/a PyObject *fast_memo;
573n/a} PicklerObject;
574n/a
575n/atypedef struct UnpicklerObject {
576n/a PyObject_HEAD
577n/a Pdata *stack; /* Pickle data stack, store unpickled objects. */
578n/a
579n/a /* The unpickler memo is just an array of PyObject *s. Using a dict
580n/a is unnecessary, since the keys are contiguous ints. */
581n/a PyObject **memo;
582n/a Py_ssize_t memo_size; /* Capacity of the memo array */
583n/a Py_ssize_t memo_len; /* Number of objects in the memo */
584n/a
585n/a PyObject *pers_func; /* persistent_load() method, can be NULL. */
586n/a
587n/a Py_buffer buffer;
588n/a char *input_buffer;
589n/a char *input_line;
590n/a Py_ssize_t input_len;
591n/a Py_ssize_t next_read_idx;
592n/a Py_ssize_t prefetched_idx; /* index of first prefetched byte */
593n/a
594n/a PyObject *read; /* read() method of the input stream. */
595n/a PyObject *readline; /* readline() method of the input stream. */
596n/a PyObject *peek; /* peek() method of the input stream, or NULL */
597n/a
598n/a char *encoding; /* Name of the encoding to be used for
599n/a decoding strings pickled using Python
600n/a 2.x. The default value is "ASCII" */
601n/a char *errors; /* Name of errors handling scheme to used when
602n/a decoding strings. The default value is
603n/a "strict". */
604n/a Py_ssize_t *marks; /* Mark stack, used for unpickling container
605n/a objects. */
606n/a Py_ssize_t num_marks; /* Number of marks in the mark stack. */
607n/a Py_ssize_t marks_size; /* Current allocated size of the mark stack. */
608n/a int proto; /* Protocol of the pickle loaded. */
609n/a int fix_imports; /* Indicate whether Unpickler should fix
610n/a the name of globals pickled by Python 2.x. */
611n/a} UnpicklerObject;
612n/a
613n/atypedef struct {
614n/a PyObject_HEAD
615n/a PicklerObject *pickler; /* Pickler whose memo table we're proxying. */
616n/a} PicklerMemoProxyObject;
617n/a
618n/atypedef struct {
619n/a PyObject_HEAD
620n/a UnpicklerObject *unpickler;
621n/a} UnpicklerMemoProxyObject;
622n/a
623n/a/* Forward declarations */
624n/astatic int save(PicklerObject *, PyObject *, int);
625n/astatic int save_reduce(PicklerObject *, PyObject *, PyObject *);
626n/astatic PyTypeObject Pickler_Type;
627n/astatic PyTypeObject Unpickler_Type;
628n/a
629n/a#include "clinic/_pickle.c.h"
630n/a
631n/a/*************************************************************************
632n/a A custom hashtable mapping void* to Python ints. This is used by the pickler
633n/a for memoization. Using a custom hashtable rather than PyDict allows us to skip
634n/a a bunch of unnecessary object creation. This makes a huge performance
635n/a difference. */
636n/a
637n/a#define MT_MINSIZE 8
638n/a#define PERTURB_SHIFT 5
639n/a
640n/a
641n/astatic PyMemoTable *
642n/aPyMemoTable_New(void)
643n/a{
644n/a PyMemoTable *memo = PyMem_MALLOC(sizeof(PyMemoTable));
645n/a if (memo == NULL) {
646n/a PyErr_NoMemory();
647n/a return NULL;
648n/a }
649n/a
650n/a memo->mt_used = 0;
651n/a memo->mt_allocated = MT_MINSIZE;
652n/a memo->mt_mask = MT_MINSIZE - 1;
653n/a memo->mt_table = PyMem_MALLOC(MT_MINSIZE * sizeof(PyMemoEntry));
654n/a if (memo->mt_table == NULL) {
655n/a PyMem_FREE(memo);
656n/a PyErr_NoMemory();
657n/a return NULL;
658n/a }
659n/a memset(memo->mt_table, 0, MT_MINSIZE * sizeof(PyMemoEntry));
660n/a
661n/a return memo;
662n/a}
663n/a
664n/astatic PyMemoTable *
665n/aPyMemoTable_Copy(PyMemoTable *self)
666n/a{
667n/a Py_ssize_t i;
668n/a PyMemoTable *new = PyMemoTable_New();
669n/a if (new == NULL)
670n/a return NULL;
671n/a
672n/a new->mt_used = self->mt_used;
673n/a new->mt_allocated = self->mt_allocated;
674n/a new->mt_mask = self->mt_mask;
675n/a /* The table we get from _New() is probably smaller than we wanted.
676n/a Free it and allocate one that's the right size. */
677n/a PyMem_FREE(new->mt_table);
678n/a new->mt_table = PyMem_NEW(PyMemoEntry, self->mt_allocated);
679n/a if (new->mt_table == NULL) {
680n/a PyMem_FREE(new);
681n/a PyErr_NoMemory();
682n/a return NULL;
683n/a }
684n/a for (i = 0; i < self->mt_allocated; i++) {
685n/a Py_XINCREF(self->mt_table[i].me_key);
686n/a }
687n/a memcpy(new->mt_table, self->mt_table,
688n/a sizeof(PyMemoEntry) * self->mt_allocated);
689n/a
690n/a return new;
691n/a}
692n/a
693n/astatic Py_ssize_t
694n/aPyMemoTable_Size(PyMemoTable *self)
695n/a{
696n/a return self->mt_used;
697n/a}
698n/a
699n/astatic int
700n/aPyMemoTable_Clear(PyMemoTable *self)
701n/a{
702n/a Py_ssize_t i = self->mt_allocated;
703n/a
704n/a while (--i >= 0) {
705n/a Py_XDECREF(self->mt_table[i].me_key);
706n/a }
707n/a self->mt_used = 0;
708n/a memset(self->mt_table, 0, self->mt_allocated * sizeof(PyMemoEntry));
709n/a return 0;
710n/a}
711n/a
712n/astatic void
713n/aPyMemoTable_Del(PyMemoTable *self)
714n/a{
715n/a if (self == NULL)
716n/a return;
717n/a PyMemoTable_Clear(self);
718n/a
719n/a PyMem_FREE(self->mt_table);
720n/a PyMem_FREE(self);
721n/a}
722n/a
723n/a/* Since entries cannot be deleted from this hashtable, _PyMemoTable_Lookup()
724n/a can be considerably simpler than dictobject.c's lookdict(). */
725n/astatic PyMemoEntry *
726n/a_PyMemoTable_Lookup(PyMemoTable *self, PyObject *key)
727n/a{
728n/a size_t i;
729n/a size_t perturb;
730n/a size_t mask = (size_t)self->mt_mask;
731n/a PyMemoEntry *table = self->mt_table;
732n/a PyMemoEntry *entry;
733n/a Py_hash_t hash = (Py_hash_t)key >> 3;
734n/a
735n/a i = hash & mask;
736n/a entry = &table[i];
737n/a if (entry->me_key == NULL || entry->me_key == key)
738n/a return entry;
739n/a
740n/a for (perturb = hash; ; perturb >>= PERTURB_SHIFT) {
741n/a i = (i << 2) + i + perturb + 1;
742n/a entry = &table[i & mask];
743n/a if (entry->me_key == NULL || entry->me_key == key)
744n/a return entry;
745n/a }
746n/a assert(0); /* Never reached */
747n/a return NULL;
748n/a}
749n/a
750n/a/* Returns -1 on failure, 0 on success. */
751n/astatic int
752n/a_PyMemoTable_ResizeTable(PyMemoTable *self, Py_ssize_t min_size)
753n/a{
754n/a PyMemoEntry *oldtable = NULL;
755n/a PyMemoEntry *oldentry, *newentry;
756n/a Py_ssize_t new_size = MT_MINSIZE;
757n/a Py_ssize_t to_process;
758n/a
759n/a assert(min_size > 0);
760n/a
761n/a /* Find the smallest valid table size >= min_size. */
762n/a while (new_size < min_size && new_size > 0)
763n/a new_size <<= 1;
764n/a if (new_size <= 0) {
765n/a PyErr_NoMemory();
766n/a return -1;
767n/a }
768n/a /* new_size needs to be a power of two. */
769n/a assert((new_size & (new_size - 1)) == 0);
770n/a
771n/a /* Allocate new table. */
772n/a oldtable = self->mt_table;
773n/a self->mt_table = PyMem_NEW(PyMemoEntry, new_size);
774n/a if (self->mt_table == NULL) {
775n/a self->mt_table = oldtable;
776n/a PyErr_NoMemory();
777n/a return -1;
778n/a }
779n/a self->mt_allocated = new_size;
780n/a self->mt_mask = new_size - 1;
781n/a memset(self->mt_table, 0, sizeof(PyMemoEntry) * new_size);
782n/a
783n/a /* Copy entries from the old table. */
784n/a to_process = self->mt_used;
785n/a for (oldentry = oldtable; to_process > 0; oldentry++) {
786n/a if (oldentry->me_key != NULL) {
787n/a to_process--;
788n/a /* newentry is a pointer to a chunk of the new
789n/a mt_table, so we're setting the key:value pair
790n/a in-place. */
791n/a newentry = _PyMemoTable_Lookup(self, oldentry->me_key);
792n/a newentry->me_key = oldentry->me_key;
793n/a newentry->me_value = oldentry->me_value;
794n/a }
795n/a }
796n/a
797n/a /* Deallocate the old table. */
798n/a PyMem_FREE(oldtable);
799n/a return 0;
800n/a}
801n/a
802n/a/* Returns NULL on failure, a pointer to the value otherwise. */
803n/astatic Py_ssize_t *
804n/aPyMemoTable_Get(PyMemoTable *self, PyObject *key)
805n/a{
806n/a PyMemoEntry *entry = _PyMemoTable_Lookup(self, key);
807n/a if (entry->me_key == NULL)
808n/a return NULL;
809n/a return &entry->me_value;
810n/a}
811n/a
812n/a/* Returns -1 on failure, 0 on success. */
813n/astatic int
814n/aPyMemoTable_Set(PyMemoTable *self, PyObject *key, Py_ssize_t value)
815n/a{
816n/a PyMemoEntry *entry;
817n/a
818n/a assert(key != NULL);
819n/a
820n/a entry = _PyMemoTable_Lookup(self, key);
821n/a if (entry->me_key != NULL) {
822n/a entry->me_value = value;
823n/a return 0;
824n/a }
825n/a Py_INCREF(key);
826n/a entry->me_key = key;
827n/a entry->me_value = value;
828n/a self->mt_used++;
829n/a
830n/a /* If we added a key, we can safely resize. Otherwise just return!
831n/a * If used >= 2/3 size, adjust size. Normally, this quaduples the size.
832n/a *
833n/a * Quadrupling the size improves average table sparseness
834n/a * (reducing collisions) at the cost of some memory. It also halves
835n/a * the number of expensive resize operations in a growing memo table.
836n/a *
837n/a * Very large memo tables (over 50K items) use doubling instead.
838n/a * This may help applications with severe memory constraints.
839n/a */
840n/a if (!(self->mt_used * 3 >= (self->mt_mask + 1) * 2))
841n/a return 0;
842n/a return _PyMemoTable_ResizeTable(self,
843n/a (self->mt_used > 50000 ? 2 : 4) * self->mt_used);
844n/a}
845n/a
846n/a#undef MT_MINSIZE
847n/a#undef PERTURB_SHIFT
848n/a
849n/a/*************************************************************************/
850n/a
851n/a
852n/astatic int
853n/a_Pickler_ClearBuffer(PicklerObject *self)
854n/a{
855n/a Py_XSETREF(self->output_buffer,
856n/a PyBytes_FromStringAndSize(NULL, self->max_output_len));
857n/a if (self->output_buffer == NULL)
858n/a return -1;
859n/a self->output_len = 0;
860n/a self->frame_start = -1;
861n/a return 0;
862n/a}
863n/a
864n/astatic void
865n/a_write_size64(char *out, size_t value)
866n/a{
867n/a size_t i;
868n/a
869n/a Py_BUILD_ASSERT(sizeof(size_t) <= 8);
870n/a
871n/a for (i = 0; i < sizeof(size_t); i++) {
872n/a out[i] = (unsigned char)((value >> (8 * i)) & 0xff);
873n/a }
874n/a for (i = sizeof(size_t); i < 8; i++) {
875n/a out[i] = 0;
876n/a }
877n/a}
878n/a
879n/astatic void
880n/a_Pickler_WriteFrameHeader(PicklerObject *self, char *qdata, size_t frame_len)
881n/a{
882n/a qdata[0] = FRAME;
883n/a _write_size64(qdata + 1, frame_len);
884n/a}
885n/a
886n/astatic int
887n/a_Pickler_CommitFrame(PicklerObject *self)
888n/a{
889n/a size_t frame_len;
890n/a char *qdata;
891n/a
892n/a if (!self->framing || self->frame_start == -1)
893n/a return 0;
894n/a frame_len = self->output_len - self->frame_start - FRAME_HEADER_SIZE;
895n/a qdata = PyBytes_AS_STRING(self->output_buffer) + self->frame_start;
896n/a _Pickler_WriteFrameHeader(self, qdata, frame_len);
897n/a self->frame_start = -1;
898n/a return 0;
899n/a}
900n/a
901n/astatic int
902n/a_Pickler_OpcodeBoundary(PicklerObject *self)
903n/a{
904n/a Py_ssize_t frame_len;
905n/a
906n/a if (!self->framing || self->frame_start == -1)
907n/a return 0;
908n/a frame_len = self->output_len - self->frame_start - FRAME_HEADER_SIZE;
909n/a if (frame_len >= FRAME_SIZE_TARGET)
910n/a return _Pickler_CommitFrame(self);
911n/a else
912n/a return 0;
913n/a}
914n/a
915n/astatic PyObject *
916n/a_Pickler_GetString(PicklerObject *self)
917n/a{
918n/a PyObject *output_buffer = self->output_buffer;
919n/a
920n/a assert(self->output_buffer != NULL);
921n/a
922n/a if (_Pickler_CommitFrame(self))
923n/a return NULL;
924n/a
925n/a self->output_buffer = NULL;
926n/a /* Resize down to exact size */
927n/a if (_PyBytes_Resize(&output_buffer, self->output_len) < 0)
928n/a return NULL;
929n/a return output_buffer;
930n/a}
931n/a
932n/astatic int
933n/a_Pickler_FlushToFile(PicklerObject *self)
934n/a{
935n/a PyObject *output, *result;
936n/a
937n/a assert(self->write != NULL);
938n/a
939n/a /* This will commit the frame first */
940n/a output = _Pickler_GetString(self);
941n/a if (output == NULL)
942n/a return -1;
943n/a
944n/a result = _Pickle_FastCall(self->write, output);
945n/a Py_XDECREF(result);
946n/a return (result == NULL) ? -1 : 0;
947n/a}
948n/a
949n/astatic Py_ssize_t
950n/a_Pickler_Write(PicklerObject *self, const char *s, Py_ssize_t data_len)
951n/a{
952n/a Py_ssize_t i, n, required;
953n/a char *buffer;
954n/a int need_new_frame;
955n/a
956n/a assert(s != NULL);
957n/a need_new_frame = (self->framing && self->frame_start == -1);
958n/a
959n/a if (need_new_frame)
960n/a n = data_len + FRAME_HEADER_SIZE;
961n/a else
962n/a n = data_len;
963n/a
964n/a required = self->output_len + n;
965n/a if (required > self->max_output_len) {
966n/a /* Make place in buffer for the pickle chunk */
967n/a if (self->output_len >= PY_SSIZE_T_MAX / 2 - n) {
968n/a PyErr_NoMemory();
969n/a return -1;
970n/a }
971n/a self->max_output_len = (self->output_len + n) / 2 * 3;
972n/a if (_PyBytes_Resize(&self->output_buffer, self->max_output_len) < 0)
973n/a return -1;
974n/a }
975n/a buffer = PyBytes_AS_STRING(self->output_buffer);
976n/a if (need_new_frame) {
977n/a /* Setup new frame */
978n/a Py_ssize_t frame_start = self->output_len;
979n/a self->frame_start = frame_start;
980n/a for (i = 0; i < FRAME_HEADER_SIZE; i++) {
981n/a /* Write an invalid value, for debugging */
982n/a buffer[frame_start + i] = 0xFE;
983n/a }
984n/a self->output_len += FRAME_HEADER_SIZE;
985n/a }
986n/a if (data_len < 8) {
987n/a /* This is faster than memcpy when the string is short. */
988n/a for (i = 0; i < data_len; i++) {
989n/a buffer[self->output_len + i] = s[i];
990n/a }
991n/a }
992n/a else {
993n/a memcpy(buffer + self->output_len, s, data_len);
994n/a }
995n/a self->output_len += data_len;
996n/a return data_len;
997n/a}
998n/a
999n/astatic PicklerObject *
1000n/a_Pickler_New(void)
1001n/a{
1002n/a PicklerObject *self;
1003n/a
1004n/a self = PyObject_GC_New(PicklerObject, &Pickler_Type);
1005n/a if (self == NULL)
1006n/a return NULL;
1007n/a
1008n/a self->pers_func = NULL;
1009n/a self->dispatch_table = NULL;
1010n/a self->write = NULL;
1011n/a self->proto = 0;
1012n/a self->bin = 0;
1013n/a self->framing = 0;
1014n/a self->frame_start = -1;
1015n/a self->fast = 0;
1016n/a self->fast_nesting = 0;
1017n/a self->fix_imports = 0;
1018n/a self->fast_memo = NULL;
1019n/a self->max_output_len = WRITE_BUF_SIZE;
1020n/a self->output_len = 0;
1021n/a
1022n/a self->memo = PyMemoTable_New();
1023n/a self->output_buffer = PyBytes_FromStringAndSize(NULL,
1024n/a self->max_output_len);
1025n/a
1026n/a if (self->memo == NULL || self->output_buffer == NULL) {
1027n/a Py_DECREF(self);
1028n/a return NULL;
1029n/a }
1030n/a return self;
1031n/a}
1032n/a
1033n/astatic int
1034n/a_Pickler_SetProtocol(PicklerObject *self, PyObject *protocol, int fix_imports)
1035n/a{
1036n/a long proto;
1037n/a
1038n/a if (protocol == NULL || protocol == Py_None) {
1039n/a proto = DEFAULT_PROTOCOL;
1040n/a }
1041n/a else {
1042n/a proto = PyLong_AsLong(protocol);
1043n/a if (proto < 0) {
1044n/a if (proto == -1 && PyErr_Occurred())
1045n/a return -1;
1046n/a proto = HIGHEST_PROTOCOL;
1047n/a }
1048n/a else if (proto > HIGHEST_PROTOCOL) {
1049n/a PyErr_Format(PyExc_ValueError, "pickle protocol must be <= %d",
1050n/a HIGHEST_PROTOCOL);
1051n/a return -1;
1052n/a }
1053n/a }
1054n/a self->proto = (int)proto;
1055n/a self->bin = proto > 0;
1056n/a self->fix_imports = fix_imports && proto < 3;
1057n/a return 0;
1058n/a}
1059n/a
1060n/a/* Returns -1 (with an exception set) on failure, 0 on success. This may
1061n/a be called once on a freshly created Pickler. */
1062n/astatic int
1063n/a_Pickler_SetOutputStream(PicklerObject *self, PyObject *file)
1064n/a{
1065n/a _Py_IDENTIFIER(write);
1066n/a assert(file != NULL);
1067n/a self->write = _PyObject_GetAttrId(file, &PyId_write);
1068n/a if (self->write == NULL) {
1069n/a if (PyErr_ExceptionMatches(PyExc_AttributeError))
1070n/a PyErr_SetString(PyExc_TypeError,
1071n/a "file must have a 'write' attribute");
1072n/a return -1;
1073n/a }
1074n/a
1075n/a return 0;
1076n/a}
1077n/a
1078n/a/* Returns the size of the input on success, -1 on failure. This takes its
1079n/a own reference to `input`. */
1080n/astatic Py_ssize_t
1081n/a_Unpickler_SetStringInput(UnpicklerObject *self, PyObject *input)
1082n/a{
1083n/a if (self->buffer.buf != NULL)
1084n/a PyBuffer_Release(&self->buffer);
1085n/a if (PyObject_GetBuffer(input, &self->buffer, PyBUF_CONTIG_RO) < 0)
1086n/a return -1;
1087n/a self->input_buffer = self->buffer.buf;
1088n/a self->input_len = self->buffer.len;
1089n/a self->next_read_idx = 0;
1090n/a self->prefetched_idx = self->input_len;
1091n/a return self->input_len;
1092n/a}
1093n/a
1094n/astatic int
1095n/abad_readline(void)
1096n/a{
1097n/a PickleState *st = _Pickle_GetGlobalState();
1098n/a PyErr_SetString(st->UnpicklingError, "pickle data was truncated");
1099n/a return -1;
1100n/a}
1101n/a
1102n/astatic int
1103n/a_Unpickler_SkipConsumed(UnpicklerObject *self)
1104n/a{
1105n/a Py_ssize_t consumed;
1106n/a PyObject *r;
1107n/a
1108n/a consumed = self->next_read_idx - self->prefetched_idx;
1109n/a if (consumed <= 0)
1110n/a return 0;
1111n/a
1112n/a assert(self->peek); /* otherwise we did something wrong */
1113n/a /* This makes a useless copy... */
1114n/a r = PyObject_CallFunction(self->read, "n", consumed);
1115n/a if (r == NULL)
1116n/a return -1;
1117n/a Py_DECREF(r);
1118n/a
1119n/a self->prefetched_idx = self->next_read_idx;
1120n/a return 0;
1121n/a}
1122n/a
1123n/astatic const Py_ssize_t READ_WHOLE_LINE = -1;
1124n/a
1125n/a/* If reading from a file, we need to only pull the bytes we need, since there
1126n/a may be multiple pickle objects arranged contiguously in the same input
1127n/a buffer.
1128n/a
1129n/a If `n` is READ_WHOLE_LINE, read a whole line. Otherwise, read up to `n`
1130n/a bytes from the input stream/buffer.
1131n/a
1132n/a Update the unpickler's input buffer with the newly-read data. Returns -1 on
1133n/a failure; on success, returns the number of bytes read from the file.
1134n/a
1135n/a On success, self->input_len will be 0; this is intentional so that when
1136n/a unpickling from a file, the "we've run out of data" code paths will trigger,
1137n/a causing the Unpickler to go back to the file for more data. Use the returned
1138n/a size to tell you how much data you can process. */
1139n/astatic Py_ssize_t
1140n/a_Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n)
1141n/a{
1142n/a PyObject *data;
1143n/a Py_ssize_t read_size;
1144n/a
1145n/a assert(self->read != NULL);
1146n/a
1147n/a if (_Unpickler_SkipConsumed(self) < 0)
1148n/a return -1;
1149n/a
1150n/a if (n == READ_WHOLE_LINE) {
1151n/a data = _PyObject_CallNoArg(self->readline);
1152n/a }
1153n/a else {
1154n/a PyObject *len;
1155n/a /* Prefetch some data without advancing the file pointer, if possible */
1156n/a if (self->peek && n < PREFETCH) {
1157n/a len = PyLong_FromSsize_t(PREFETCH);
1158n/a if (len == NULL)
1159n/a return -1;
1160n/a data = _Pickle_FastCall(self->peek, len);
1161n/a if (data == NULL) {
1162n/a if (!PyErr_ExceptionMatches(PyExc_NotImplementedError))
1163n/a return -1;
1164n/a /* peek() is probably not supported by the given file object */
1165n/a PyErr_Clear();
1166n/a Py_CLEAR(self->peek);
1167n/a }
1168n/a else {
1169n/a read_size = _Unpickler_SetStringInput(self, data);
1170n/a Py_DECREF(data);
1171n/a self->prefetched_idx = 0;
1172n/a if (n <= read_size)
1173n/a return n;
1174n/a }
1175n/a }
1176n/a len = PyLong_FromSsize_t(n);
1177n/a if (len == NULL)
1178n/a return -1;
1179n/a data = _Pickle_FastCall(self->read, len);
1180n/a }
1181n/a if (data == NULL)
1182n/a return -1;
1183n/a
1184n/a read_size = _Unpickler_SetStringInput(self, data);
1185n/a Py_DECREF(data);
1186n/a return read_size;
1187n/a}
1188n/a
1189n/a/* Don't call it directly: use _Unpickler_Read() */
1190n/astatic Py_ssize_t
1191n/a_Unpickler_ReadImpl(UnpicklerObject *self, char **s, Py_ssize_t n)
1192n/a{
1193n/a Py_ssize_t num_read;
1194n/a
1195n/a *s = NULL;
1196n/a if (self->next_read_idx > PY_SSIZE_T_MAX - n) {
1197n/a PickleState *st = _Pickle_GetGlobalState();
1198n/a PyErr_SetString(st->UnpicklingError,
1199n/a "read would overflow (invalid bytecode)");
1200n/a return -1;
1201n/a }
1202n/a
1203n/a /* This case is handled by the _Unpickler_Read() macro for efficiency */
1204n/a assert(self->next_read_idx + n > self->input_len);
1205n/a
1206n/a if (!self->read)
1207n/a return bad_readline();
1208n/a
1209n/a num_read = _Unpickler_ReadFromFile(self, n);
1210n/a if (num_read < 0)
1211n/a return -1;
1212n/a if (num_read < n)
1213n/a return bad_readline();
1214n/a *s = self->input_buffer;
1215n/a self->next_read_idx = n;
1216n/a return n;
1217n/a}
1218n/a
1219n/a/* Read `n` bytes from the unpickler's data source, storing the result in `*s`.
1220n/a
1221n/a This should be used for all data reads, rather than accessing the unpickler's
1222n/a input buffer directly. This method deals correctly with reading from input
1223n/a streams, which the input buffer doesn't deal with.
1224n/a
1225n/a Note that when reading from a file-like object, self->next_read_idx won't
1226n/a be updated (it should remain at 0 for the entire unpickling process). You
1227n/a should use this function's return value to know how many bytes you can
1228n/a consume.
1229n/a
1230n/a Returns -1 (with an exception set) on failure. On success, return the
1231n/a number of chars read. */
1232n/a#define _Unpickler_Read(self, s, n) \
1233n/a (((n) <= (self)->input_len - (self)->next_read_idx) \
1234n/a ? (*(s) = (self)->input_buffer + (self)->next_read_idx, \
1235n/a (self)->next_read_idx += (n), \
1236n/a (n)) \
1237n/a : _Unpickler_ReadImpl(self, (s), (n)))
1238n/a
1239n/astatic Py_ssize_t
1240n/a_Unpickler_CopyLine(UnpicklerObject *self, char *line, Py_ssize_t len,
1241n/a char **result)
1242n/a{
1243n/a char *input_line = PyMem_Realloc(self->input_line, len + 1);
1244n/a if (input_line == NULL) {
1245n/a PyErr_NoMemory();
1246n/a return -1;
1247n/a }
1248n/a
1249n/a memcpy(input_line, line, len);
1250n/a input_line[len] = '\0';
1251n/a self->input_line = input_line;
1252n/a *result = self->input_line;
1253n/a return len;
1254n/a}
1255n/a
1256n/a/* Read a line from the input stream/buffer. If we run off the end of the input
1257n/a before hitting \n, raise an error.
1258n/a
1259n/a Returns the number of chars read, or -1 on failure. */
1260n/astatic Py_ssize_t
1261n/a_Unpickler_Readline(UnpicklerObject *self, char **result)
1262n/a{
1263n/a Py_ssize_t i, num_read;
1264n/a
1265n/a for (i = self->next_read_idx; i < self->input_len; i++) {
1266n/a if (self->input_buffer[i] == '\n') {
1267n/a char *line_start = self->input_buffer + self->next_read_idx;
1268n/a num_read = i - self->next_read_idx + 1;
1269n/a self->next_read_idx = i + 1;
1270n/a return _Unpickler_CopyLine(self, line_start, num_read, result);
1271n/a }
1272n/a }
1273n/a if (!self->read)
1274n/a return bad_readline();
1275n/a
1276n/a num_read = _Unpickler_ReadFromFile(self, READ_WHOLE_LINE);
1277n/a if (num_read < 0)
1278n/a return -1;
1279n/a if (num_read == 0 || self->input_buffer[num_read - 1] != '\n')
1280n/a return bad_readline();
1281n/a self->next_read_idx = num_read;
1282n/a return _Unpickler_CopyLine(self, self->input_buffer, num_read, result);
1283n/a}
1284n/a
1285n/a/* Returns -1 (with an exception set) on failure, 0 on success. The memo array
1286n/a will be modified in place. */
1287n/astatic int
1288n/a_Unpickler_ResizeMemoList(UnpicklerObject *self, Py_ssize_t new_size)
1289n/a{
1290n/a Py_ssize_t i;
1291n/a
1292n/a assert(new_size > self->memo_size);
1293n/a
1294n/a PyMem_RESIZE(self->memo, PyObject *, new_size);
1295n/a if (self->memo == NULL) {
1296n/a PyErr_NoMemory();
1297n/a return -1;
1298n/a }
1299n/a for (i = self->memo_size; i < new_size; i++)
1300n/a self->memo[i] = NULL;
1301n/a self->memo_size = new_size;
1302n/a return 0;
1303n/a}
1304n/a
1305n/a/* Returns NULL if idx is out of bounds. */
1306n/astatic PyObject *
1307n/a_Unpickler_MemoGet(UnpicklerObject *self, Py_ssize_t idx)
1308n/a{
1309n/a if (idx < 0 || idx >= self->memo_size)
1310n/a return NULL;
1311n/a
1312n/a return self->memo[idx];
1313n/a}
1314n/a
1315n/a/* Returns -1 (with an exception set) on failure, 0 on success.
1316n/a This takes its own reference to `value`. */
1317n/astatic int
1318n/a_Unpickler_MemoPut(UnpicklerObject *self, Py_ssize_t idx, PyObject *value)
1319n/a{
1320n/a PyObject *old_item;
1321n/a
1322n/a if (idx >= self->memo_size) {
1323n/a if (_Unpickler_ResizeMemoList(self, idx * 2) < 0)
1324n/a return -1;
1325n/a assert(idx < self->memo_size);
1326n/a }
1327n/a Py_INCREF(value);
1328n/a old_item = self->memo[idx];
1329n/a self->memo[idx] = value;
1330n/a if (old_item != NULL) {
1331n/a Py_DECREF(old_item);
1332n/a }
1333n/a else {
1334n/a self->memo_len++;
1335n/a }
1336n/a return 0;
1337n/a}
1338n/a
1339n/astatic PyObject **
1340n/a_Unpickler_NewMemo(Py_ssize_t new_size)
1341n/a{
1342n/a PyObject **memo = PyMem_NEW(PyObject *, new_size);
1343n/a if (memo == NULL) {
1344n/a PyErr_NoMemory();
1345n/a return NULL;
1346n/a }
1347n/a memset(memo, 0, new_size * sizeof(PyObject *));
1348n/a return memo;
1349n/a}
1350n/a
1351n/a/* Free the unpickler's memo, taking care to decref any items left in it. */
1352n/astatic void
1353n/a_Unpickler_MemoCleanup(UnpicklerObject *self)
1354n/a{
1355n/a Py_ssize_t i;
1356n/a PyObject **memo = self->memo;
1357n/a
1358n/a if (self->memo == NULL)
1359n/a return;
1360n/a self->memo = NULL;
1361n/a i = self->memo_size;
1362n/a while (--i >= 0) {
1363n/a Py_XDECREF(memo[i]);
1364n/a }
1365n/a PyMem_FREE(memo);
1366n/a}
1367n/a
1368n/astatic UnpicklerObject *
1369n/a_Unpickler_New(void)
1370n/a{
1371n/a UnpicklerObject *self;
1372n/a
1373n/a self = PyObject_GC_New(UnpicklerObject, &Unpickler_Type);
1374n/a if (self == NULL)
1375n/a return NULL;
1376n/a
1377n/a self->pers_func = NULL;
1378n/a self->input_buffer = NULL;
1379n/a self->input_line = NULL;
1380n/a self->input_len = 0;
1381n/a self->next_read_idx = 0;
1382n/a self->prefetched_idx = 0;
1383n/a self->read = NULL;
1384n/a self->readline = NULL;
1385n/a self->peek = NULL;
1386n/a self->encoding = NULL;
1387n/a self->errors = NULL;
1388n/a self->marks = NULL;
1389n/a self->num_marks = 0;
1390n/a self->marks_size = 0;
1391n/a self->proto = 0;
1392n/a self->fix_imports = 0;
1393n/a memset(&self->buffer, 0, sizeof(Py_buffer));
1394n/a self->memo_size = 32;
1395n/a self->memo_len = 0;
1396n/a self->memo = _Unpickler_NewMemo(self->memo_size);
1397n/a self->stack = (Pdata *)Pdata_New();
1398n/a
1399n/a if (self->memo == NULL || self->stack == NULL) {
1400n/a Py_DECREF(self);
1401n/a return NULL;
1402n/a }
1403n/a
1404n/a return self;
1405n/a}
1406n/a
1407n/a/* Returns -1 (with an exception set) on failure, 0 on success. This may
1408n/a be called once on a freshly created Pickler. */
1409n/astatic int
1410n/a_Unpickler_SetInputStream(UnpicklerObject *self, PyObject *file)
1411n/a{
1412n/a _Py_IDENTIFIER(peek);
1413n/a _Py_IDENTIFIER(read);
1414n/a _Py_IDENTIFIER(readline);
1415n/a
1416n/a self->peek = _PyObject_GetAttrId(file, &PyId_peek);
1417n/a if (self->peek == NULL) {
1418n/a if (PyErr_ExceptionMatches(PyExc_AttributeError))
1419n/a PyErr_Clear();
1420n/a else
1421n/a return -1;
1422n/a }
1423n/a self->read = _PyObject_GetAttrId(file, &PyId_read);
1424n/a self->readline = _PyObject_GetAttrId(file, &PyId_readline);
1425n/a if (self->readline == NULL || self->read == NULL) {
1426n/a if (PyErr_ExceptionMatches(PyExc_AttributeError))
1427n/a PyErr_SetString(PyExc_TypeError,
1428n/a "file must have 'read' and 'readline' attributes");
1429n/a Py_CLEAR(self->read);
1430n/a Py_CLEAR(self->readline);
1431n/a Py_CLEAR(self->peek);
1432n/a return -1;
1433n/a }
1434n/a return 0;
1435n/a}
1436n/a
1437n/a/* Returns -1 (with an exception set) on failure, 0 on success. This may
1438n/a be called once on a freshly created Pickler. */
1439n/astatic int
1440n/a_Unpickler_SetInputEncoding(UnpicklerObject *self,
1441n/a const char *encoding,
1442n/a const char *errors)
1443n/a{
1444n/a if (encoding == NULL)
1445n/a encoding = "ASCII";
1446n/a if (errors == NULL)
1447n/a errors = "strict";
1448n/a
1449n/a self->encoding = _PyMem_Strdup(encoding);
1450n/a self->errors = _PyMem_Strdup(errors);
1451n/a if (self->encoding == NULL || self->errors == NULL) {
1452n/a PyErr_NoMemory();
1453n/a return -1;
1454n/a }
1455n/a return 0;
1456n/a}
1457n/a
1458n/a/* Generate a GET opcode for an object stored in the memo. */
1459n/astatic int
1460n/amemo_get(PicklerObject *self, PyObject *key)
1461n/a{
1462n/a Py_ssize_t *value;
1463n/a char pdata[30];
1464n/a Py_ssize_t len;
1465n/a
1466n/a value = PyMemoTable_Get(self->memo, key);
1467n/a if (value == NULL) {
1468n/a PyErr_SetObject(PyExc_KeyError, key);
1469n/a return -1;
1470n/a }
1471n/a
1472n/a if (!self->bin) {
1473n/a pdata[0] = GET;
1474n/a PyOS_snprintf(pdata + 1, sizeof(pdata) - 1,
1475n/a "%" PY_FORMAT_SIZE_T "d\n", *value);
1476n/a len = strlen(pdata);
1477n/a }
1478n/a else {
1479n/a if (*value < 256) {
1480n/a pdata[0] = BINGET;
1481n/a pdata[1] = (unsigned char)(*value & 0xff);
1482n/a len = 2;
1483n/a }
1484n/a else if ((size_t)*value <= 0xffffffffUL) {
1485n/a pdata[0] = LONG_BINGET;
1486n/a pdata[1] = (unsigned char)(*value & 0xff);
1487n/a pdata[2] = (unsigned char)((*value >> 8) & 0xff);
1488n/a pdata[3] = (unsigned char)((*value >> 16) & 0xff);
1489n/a pdata[4] = (unsigned char)((*value >> 24) & 0xff);
1490n/a len = 5;
1491n/a }
1492n/a else { /* unlikely */
1493n/a PickleState *st = _Pickle_GetGlobalState();
1494n/a PyErr_SetString(st->PicklingError,
1495n/a "memo id too large for LONG_BINGET");
1496n/a return -1;
1497n/a }
1498n/a }
1499n/a
1500n/a if (_Pickler_Write(self, pdata, len) < 0)
1501n/a return -1;
1502n/a
1503n/a return 0;
1504n/a}
1505n/a
1506n/a/* Store an object in the memo, assign it a new unique ID based on the number
1507n/a of objects currently stored in the memo and generate a PUT opcode. */
1508n/astatic int
1509n/amemo_put(PicklerObject *self, PyObject *obj)
1510n/a{
1511n/a char pdata[30];
1512n/a Py_ssize_t len;
1513n/a Py_ssize_t idx;
1514n/a
1515n/a const char memoize_op = MEMOIZE;
1516n/a
1517n/a if (self->fast)
1518n/a return 0;
1519n/a
1520n/a idx = PyMemoTable_Size(self->memo);
1521n/a if (PyMemoTable_Set(self->memo, obj, idx) < 0)
1522n/a return -1;
1523n/a
1524n/a if (self->proto >= 4) {
1525n/a if (_Pickler_Write(self, &memoize_op, 1) < 0)
1526n/a return -1;
1527n/a return 0;
1528n/a }
1529n/a else if (!self->bin) {
1530n/a pdata[0] = PUT;
1531n/a PyOS_snprintf(pdata + 1, sizeof(pdata) - 1,
1532n/a "%" PY_FORMAT_SIZE_T "d\n", idx);
1533n/a len = strlen(pdata);
1534n/a }
1535n/a else {
1536n/a if (idx < 256) {
1537n/a pdata[0] = BINPUT;
1538n/a pdata[1] = (unsigned char)idx;
1539n/a len = 2;
1540n/a }
1541n/a else if ((size_t)idx <= 0xffffffffUL) {
1542n/a pdata[0] = LONG_BINPUT;
1543n/a pdata[1] = (unsigned char)(idx & 0xff);
1544n/a pdata[2] = (unsigned char)((idx >> 8) & 0xff);
1545n/a pdata[3] = (unsigned char)((idx >> 16) & 0xff);
1546n/a pdata[4] = (unsigned char)((idx >> 24) & 0xff);
1547n/a len = 5;
1548n/a }
1549n/a else { /* unlikely */
1550n/a PickleState *st = _Pickle_GetGlobalState();
1551n/a PyErr_SetString(st->PicklingError,
1552n/a "memo id too large for LONG_BINPUT");
1553n/a return -1;
1554n/a }
1555n/a }
1556n/a if (_Pickler_Write(self, pdata, len) < 0)
1557n/a return -1;
1558n/a
1559n/a return 0;
1560n/a}
1561n/a
1562n/astatic PyObject *
1563n/aget_dotted_path(PyObject *obj, PyObject *name)
1564n/a{
1565n/a _Py_static_string(PyId_dot, ".");
1566n/a PyObject *dotted_path;
1567n/a Py_ssize_t i, n;
1568n/a
1569n/a dotted_path = PyUnicode_Split(name, _PyUnicode_FromId(&PyId_dot), -1);
1570n/a if (dotted_path == NULL)
1571n/a return NULL;
1572n/a n = PyList_GET_SIZE(dotted_path);
1573n/a assert(n >= 1);
1574n/a for (i = 0; i < n; i++) {
1575n/a PyObject *subpath = PyList_GET_ITEM(dotted_path, i);
1576n/a if (_PyUnicode_EqualToASCIIString(subpath, "<locals>")) {
1577n/a if (obj == NULL)
1578n/a PyErr_Format(PyExc_AttributeError,
1579n/a "Can't pickle local object %R", name);
1580n/a else
1581n/a PyErr_Format(PyExc_AttributeError,
1582n/a "Can't pickle local attribute %R on %R", name, obj);
1583n/a Py_DECREF(dotted_path);
1584n/a return NULL;
1585n/a }
1586n/a }
1587n/a return dotted_path;
1588n/a}
1589n/a
1590n/astatic PyObject *
1591n/aget_deep_attribute(PyObject *obj, PyObject *names, PyObject **pparent)
1592n/a{
1593n/a Py_ssize_t i, n;
1594n/a PyObject *parent = NULL;
1595n/a
1596n/a assert(PyList_CheckExact(names));
1597n/a Py_INCREF(obj);
1598n/a n = PyList_GET_SIZE(names);
1599n/a for (i = 0; i < n; i++) {
1600n/a PyObject *name = PyList_GET_ITEM(names, i);
1601n/a Py_XDECREF(parent);
1602n/a parent = obj;
1603n/a obj = PyObject_GetAttr(parent, name);
1604n/a if (obj == NULL) {
1605n/a Py_DECREF(parent);
1606n/a return NULL;
1607n/a }
1608n/a }
1609n/a if (pparent != NULL)
1610n/a *pparent = parent;
1611n/a else
1612n/a Py_XDECREF(parent);
1613n/a return obj;
1614n/a}
1615n/a
1616n/astatic void
1617n/areformat_attribute_error(PyObject *obj, PyObject *name)
1618n/a{
1619n/a if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
1620n/a PyErr_Clear();
1621n/a PyErr_Format(PyExc_AttributeError,
1622n/a "Can't get attribute %R on %R", name, obj);
1623n/a }
1624n/a}
1625n/a
1626n/a
1627n/astatic PyObject *
1628n/agetattribute(PyObject *obj, PyObject *name, int allow_qualname)
1629n/a{
1630n/a PyObject *dotted_path, *attr;
1631n/a
1632n/a if (allow_qualname) {
1633n/a dotted_path = get_dotted_path(obj, name);
1634n/a if (dotted_path == NULL)
1635n/a return NULL;
1636n/a attr = get_deep_attribute(obj, dotted_path, NULL);
1637n/a Py_DECREF(dotted_path);
1638n/a }
1639n/a else
1640n/a attr = PyObject_GetAttr(obj, name);
1641n/a if (attr == NULL)
1642n/a reformat_attribute_error(obj, name);
1643n/a return attr;
1644n/a}
1645n/a
1646n/astatic PyObject *
1647n/awhichmodule(PyObject *global, PyObject *dotted_path)
1648n/a{
1649n/a PyObject *module_name;
1650n/a PyObject *modules_dict;
1651n/a PyObject *module;
1652n/a Py_ssize_t i;
1653n/a _Py_IDENTIFIER(__module__);
1654n/a _Py_IDENTIFIER(modules);
1655n/a _Py_IDENTIFIER(__main__);
1656n/a
1657n/a module_name = _PyObject_GetAttrId(global, &PyId___module__);
1658n/a
1659n/a if (module_name == NULL) {
1660n/a if (!PyErr_ExceptionMatches(PyExc_AttributeError))
1661n/a return NULL;
1662n/a PyErr_Clear();
1663n/a }
1664n/a else {
1665n/a /* In some rare cases (e.g., bound methods of extension types),
1666n/a __module__ can be None. If it is so, then search sys.modules for
1667n/a the module of global. */
1668n/a if (module_name != Py_None)
1669n/a return module_name;
1670n/a Py_CLEAR(module_name);
1671n/a }
1672n/a assert(module_name == NULL);
1673n/a
1674n/a /* Fallback on walking sys.modules */
1675n/a modules_dict = _PySys_GetObjectId(&PyId_modules);
1676n/a if (modules_dict == NULL) {
1677n/a PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules");
1678n/a return NULL;
1679n/a }
1680n/a
1681n/a i = 0;
1682n/a while (PyDict_Next(modules_dict, &i, &module_name, &module)) {
1683n/a PyObject *candidate;
1684n/a if (PyUnicode_Check(module_name) &&
1685n/a _PyUnicode_EqualToASCIIString(module_name, "__main__"))
1686n/a continue;
1687n/a if (module == Py_None)
1688n/a continue;
1689n/a
1690n/a candidate = get_deep_attribute(module, dotted_path, NULL);
1691n/a if (candidate == NULL) {
1692n/a if (!PyErr_ExceptionMatches(PyExc_AttributeError))
1693n/a return NULL;
1694n/a PyErr_Clear();
1695n/a continue;
1696n/a }
1697n/a
1698n/a if (candidate == global) {
1699n/a Py_INCREF(module_name);
1700n/a Py_DECREF(candidate);
1701n/a return module_name;
1702n/a }
1703n/a Py_DECREF(candidate);
1704n/a }
1705n/a
1706n/a /* If no module is found, use __main__. */
1707n/a module_name = _PyUnicode_FromId(&PyId___main__);
1708n/a Py_INCREF(module_name);
1709n/a return module_name;
1710n/a}
1711n/a
1712n/a/* fast_save_enter() and fast_save_leave() are guards against recursive
1713n/a objects when Pickler is used with the "fast mode" (i.e., with object
1714n/a memoization disabled). If the nesting of a list or dict object exceed
1715n/a FAST_NESTING_LIMIT, these guards will start keeping an internal
1716n/a reference to the seen list or dict objects and check whether these objects
1717n/a are recursive. These are not strictly necessary, since save() has a
1718n/a hard-coded recursion limit, but they give a nicer error message than the
1719n/a typical RuntimeError. */
1720n/astatic int
1721n/afast_save_enter(PicklerObject *self, PyObject *obj)
1722n/a{
1723n/a /* if fast_nesting < 0, we're doing an error exit. */
1724n/a if (++self->fast_nesting >= FAST_NESTING_LIMIT) {
1725n/a PyObject *key = NULL;
1726n/a if (self->fast_memo == NULL) {
1727n/a self->fast_memo = PyDict_New();
1728n/a if (self->fast_memo == NULL) {
1729n/a self->fast_nesting = -1;
1730n/a return 0;
1731n/a }
1732n/a }
1733n/a key = PyLong_FromVoidPtr(obj);
1734n/a if (key == NULL)
1735n/a return 0;
1736n/a if (PyDict_GetItemWithError(self->fast_memo, key)) {
1737n/a Py_DECREF(key);
1738n/a PyErr_Format(PyExc_ValueError,
1739n/a "fast mode: can't pickle cyclic objects "
1740n/a "including object type %.200s at %p",
1741n/a obj->ob_type->tp_name, obj);
1742n/a self->fast_nesting = -1;
1743n/a return 0;
1744n/a }
1745n/a if (PyErr_Occurred()) {
1746n/a return 0;
1747n/a }
1748n/a if (PyDict_SetItem(self->fast_memo, key, Py_None) < 0) {
1749n/a Py_DECREF(key);
1750n/a self->fast_nesting = -1;
1751n/a return 0;
1752n/a }
1753n/a Py_DECREF(key);
1754n/a }
1755n/a return 1;
1756n/a}
1757n/a
1758n/astatic int
1759n/afast_save_leave(PicklerObject *self, PyObject *obj)
1760n/a{
1761n/a if (self->fast_nesting-- >= FAST_NESTING_LIMIT) {
1762n/a PyObject *key = PyLong_FromVoidPtr(obj);
1763n/a if (key == NULL)
1764n/a return 0;
1765n/a if (PyDict_DelItem(self->fast_memo, key) < 0) {
1766n/a Py_DECREF(key);
1767n/a return 0;
1768n/a }
1769n/a Py_DECREF(key);
1770n/a }
1771n/a return 1;
1772n/a}
1773n/a
1774n/astatic int
1775n/asave_none(PicklerObject *self, PyObject *obj)
1776n/a{
1777n/a const char none_op = NONE;
1778n/a if (_Pickler_Write(self, &none_op, 1) < 0)
1779n/a return -1;
1780n/a
1781n/a return 0;
1782n/a}
1783n/a
1784n/astatic int
1785n/asave_bool(PicklerObject *self, PyObject *obj)
1786n/a{
1787n/a if (self->proto >= 2) {
1788n/a const char bool_op = (obj == Py_True) ? NEWTRUE : NEWFALSE;
1789n/a if (_Pickler_Write(self, &bool_op, 1) < 0)
1790n/a return -1;
1791n/a }
1792n/a else {
1793n/a /* These aren't opcodes -- they're ways to pickle bools before protocol 2
1794n/a * so that unpicklers written before bools were introduced unpickle them
1795n/a * as ints, but unpicklers after can recognize that bools were intended.
1796n/a * Note that protocol 2 added direct ways to pickle bools.
1797n/a */
1798n/a const char *bool_str = (obj == Py_True) ? "I01\n" : "I00\n";
1799n/a if (_Pickler_Write(self, bool_str, strlen(bool_str)) < 0)
1800n/a return -1;
1801n/a }
1802n/a return 0;
1803n/a}
1804n/a
1805n/astatic int
1806n/asave_long(PicklerObject *self, PyObject *obj)
1807n/a{
1808n/a PyObject *repr = NULL;
1809n/a Py_ssize_t size;
1810n/a long val;
1811n/a int status = 0;
1812n/a
1813n/a const char long_op = LONG;
1814n/a
1815n/a val= PyLong_AsLong(obj);
1816n/a if (val == -1 && PyErr_Occurred()) {
1817n/a /* out of range for int pickling */
1818n/a PyErr_Clear();
1819n/a }
1820n/a else if (self->bin &&
1821n/a (sizeof(long) <= 4 ||
1822n/a (val <= 0x7fffffffL && val >= (-0x7fffffffL - 1)))) {
1823n/a /* result fits in a signed 4-byte integer.
1824n/a
1825n/a Note: we can't use -0x80000000L in the above condition because some
1826n/a compilers (e.g., MSVC) will promote 0x80000000L to an unsigned type
1827n/a before applying the unary minus when sizeof(long) <= 4. The
1828n/a resulting value stays unsigned which is commonly not what we want,
1829n/a so MSVC happily warns us about it. However, that result would have
1830n/a been fine because we guard for sizeof(long) <= 4 which turns the
1831n/a condition true in that particular case. */
1832n/a char pdata[32];
1833n/a Py_ssize_t len = 0;
1834n/a
1835n/a pdata[1] = (unsigned char)(val & 0xff);
1836n/a pdata[2] = (unsigned char)((val >> 8) & 0xff);
1837n/a pdata[3] = (unsigned char)((val >> 16) & 0xff);
1838n/a pdata[4] = (unsigned char)((val >> 24) & 0xff);
1839n/a
1840n/a if ((pdata[4] == 0) && (pdata[3] == 0)) {
1841n/a if (pdata[2] == 0) {
1842n/a pdata[0] = BININT1;
1843n/a len = 2;
1844n/a }
1845n/a else {
1846n/a pdata[0] = BININT2;
1847n/a len = 3;
1848n/a }
1849n/a }
1850n/a else {
1851n/a pdata[0] = BININT;
1852n/a len = 5;
1853n/a }
1854n/a
1855n/a if (_Pickler_Write(self, pdata, len) < 0)
1856n/a return -1;
1857n/a
1858n/a return 0;
1859n/a }
1860n/a
1861n/a if (self->proto >= 2) {
1862n/a /* Linear-time pickling. */
1863n/a size_t nbits;
1864n/a size_t nbytes;
1865n/a unsigned char *pdata;
1866n/a char header[5];
1867n/a int i;
1868n/a int sign = _PyLong_Sign(obj);
1869n/a
1870n/a if (sign == 0) {
1871n/a header[0] = LONG1;
1872n/a header[1] = 0; /* It's 0 -- an empty bytestring. */
1873n/a if (_Pickler_Write(self, header, 2) < 0)
1874n/a goto error;
1875n/a return 0;
1876n/a }
1877n/a nbits = _PyLong_NumBits(obj);
1878n/a if (nbits == (size_t)-1 && PyErr_Occurred())
1879n/a goto error;
1880n/a /* How many bytes do we need? There are nbits >> 3 full
1881n/a * bytes of data, and nbits & 7 leftover bits. If there
1882n/a * are any leftover bits, then we clearly need another
1883n/a * byte. Wnat's not so obvious is that we *probably*
1884n/a * need another byte even if there aren't any leftovers:
1885n/a * the most-significant bit of the most-significant byte
1886n/a * acts like a sign bit, and it's usually got a sense
1887n/a * opposite of the one we need. The exception is ints
1888n/a * of the form -(2**(8*j-1)) for j > 0. Such an int is
1889n/a * its own 256's-complement, so has the right sign bit
1890n/a * even without the extra byte. That's a pain to check
1891n/a * for in advance, though, so we always grab an extra
1892n/a * byte at the start, and cut it back later if possible.
1893n/a */
1894n/a nbytes = (nbits >> 3) + 1;
1895n/a if (nbytes > 0x7fffffffL) {
1896n/a PyErr_SetString(PyExc_OverflowError,
1897n/a "int too large to pickle");
1898n/a goto error;
1899n/a }
1900n/a repr = PyBytes_FromStringAndSize(NULL, (Py_ssize_t)nbytes);
1901n/a if (repr == NULL)
1902n/a goto error;
1903n/a pdata = (unsigned char *)PyBytes_AS_STRING(repr);
1904n/a i = _PyLong_AsByteArray((PyLongObject *)obj,
1905n/a pdata, nbytes,
1906n/a 1 /* little endian */ , 1 /* signed */ );
1907n/a if (i < 0)
1908n/a goto error;
1909n/a /* If the int is negative, this may be a byte more than
1910n/a * needed. This is so iff the MSB is all redundant sign
1911n/a * bits.
1912n/a */
1913n/a if (sign < 0 &&
1914n/a nbytes > 1 &&
1915n/a pdata[nbytes - 1] == 0xff &&
1916n/a (pdata[nbytes - 2] & 0x80) != 0) {
1917n/a nbytes--;
1918n/a }
1919n/a
1920n/a if (nbytes < 256) {
1921n/a header[0] = LONG1;
1922n/a header[1] = (unsigned char)nbytes;
1923n/a size = 2;
1924n/a }
1925n/a else {
1926n/a header[0] = LONG4;
1927n/a size = (Py_ssize_t) nbytes;
1928n/a for (i = 1; i < 5; i++) {
1929n/a header[i] = (unsigned char)(size & 0xff);
1930n/a size >>= 8;
1931n/a }
1932n/a size = 5;
1933n/a }
1934n/a if (_Pickler_Write(self, header, size) < 0 ||
1935n/a _Pickler_Write(self, (char *)pdata, (int)nbytes) < 0)
1936n/a goto error;
1937n/a }
1938n/a else {
1939n/a const char *string;
1940n/a
1941n/a /* proto < 2: write the repr and newline. This is quadratic-time (in
1942n/a the number of digits), in both directions. We add a trailing 'L'
1943n/a to the repr, for compatibility with Python 2.x. */
1944n/a
1945n/a repr = PyObject_Repr(obj);
1946n/a if (repr == NULL)
1947n/a goto error;
1948n/a
1949n/a string = PyUnicode_AsUTF8AndSize(repr, &size);
1950n/a if (string == NULL)
1951n/a goto error;
1952n/a
1953n/a if (_Pickler_Write(self, &long_op, 1) < 0 ||
1954n/a _Pickler_Write(self, string, size) < 0 ||
1955n/a _Pickler_Write(self, "L\n", 2) < 0)
1956n/a goto error;
1957n/a }
1958n/a
1959n/a if (0) {
1960n/a error:
1961n/a status = -1;
1962n/a }
1963n/a Py_XDECREF(repr);
1964n/a
1965n/a return status;
1966n/a}
1967n/a
1968n/astatic int
1969n/asave_float(PicklerObject *self, PyObject *obj)
1970n/a{
1971n/a double x = PyFloat_AS_DOUBLE((PyFloatObject *)obj);
1972n/a
1973n/a if (self->bin) {
1974n/a char pdata[9];
1975n/a pdata[0] = BINFLOAT;
1976n/a if (_PyFloat_Pack8(x, (unsigned char *)&pdata[1], 0) < 0)
1977n/a return -1;
1978n/a if (_Pickler_Write(self, pdata, 9) < 0)
1979n/a return -1;
1980n/a }
1981n/a else {
1982n/a int result = -1;
1983n/a char *buf = NULL;
1984n/a char op = FLOAT;
1985n/a
1986n/a if (_Pickler_Write(self, &op, 1) < 0)
1987n/a goto done;
1988n/a
1989n/a buf = PyOS_double_to_string(x, 'r', 0, Py_DTSF_ADD_DOT_0, NULL);
1990n/a if (!buf) {
1991n/a PyErr_NoMemory();
1992n/a goto done;
1993n/a }
1994n/a
1995n/a if (_Pickler_Write(self, buf, strlen(buf)) < 0)
1996n/a goto done;
1997n/a
1998n/a if (_Pickler_Write(self, "\n", 1) < 0)
1999n/a goto done;
2000n/a
2001n/a result = 0;
2002n/adone:
2003n/a PyMem_Free(buf);
2004n/a return result;
2005n/a }
2006n/a
2007n/a return 0;
2008n/a}
2009n/a
2010n/astatic int
2011n/asave_bytes(PicklerObject *self, PyObject *obj)
2012n/a{
2013n/a if (self->proto < 3) {
2014n/a /* Older pickle protocols do not have an opcode for pickling bytes
2015n/a objects. Therefore, we need to fake the copy protocol (i.e.,
2016n/a the __reduce__ method) to permit bytes object unpickling.
2017n/a
2018n/a Here we use a hack to be compatible with Python 2. Since in Python
2019n/a 2 'bytes' is just an alias for 'str' (which has different
2020n/a parameters than the actual bytes object), we use codecs.encode
2021n/a to create the appropriate 'str' object when unpickled using
2022n/a Python 2 *and* the appropriate 'bytes' object when unpickled
2023n/a using Python 3. Again this is a hack and we don't need to do this
2024n/a with newer protocols. */
2025n/a PyObject *reduce_value = NULL;
2026n/a int status;
2027n/a
2028n/a if (PyBytes_GET_SIZE(obj) == 0) {
2029n/a reduce_value = Py_BuildValue("(O())", (PyObject*)&PyBytes_Type);
2030n/a }
2031n/a else {
2032n/a PickleState *st = _Pickle_GetGlobalState();
2033n/a PyObject *unicode_str =
2034n/a PyUnicode_DecodeLatin1(PyBytes_AS_STRING(obj),
2035n/a PyBytes_GET_SIZE(obj),
2036n/a "strict");
2037n/a _Py_IDENTIFIER(latin1);
2038n/a
2039n/a if (unicode_str == NULL)
2040n/a return -1;
2041n/a reduce_value = Py_BuildValue("(O(OO))",
2042n/a st->codecs_encode, unicode_str,
2043n/a _PyUnicode_FromId(&PyId_latin1));
2044n/a Py_DECREF(unicode_str);
2045n/a }
2046n/a
2047n/a if (reduce_value == NULL)
2048n/a return -1;
2049n/a
2050n/a /* save_reduce() will memoize the object automatically. */
2051n/a status = save_reduce(self, reduce_value, obj);
2052n/a Py_DECREF(reduce_value);
2053n/a return status;
2054n/a }
2055n/a else {
2056n/a Py_ssize_t size;
2057n/a char header[9];
2058n/a Py_ssize_t len;
2059n/a
2060n/a size = PyBytes_GET_SIZE(obj);
2061n/a if (size < 0)
2062n/a return -1;
2063n/a
2064n/a if (size <= 0xff) {
2065n/a header[0] = SHORT_BINBYTES;
2066n/a header[1] = (unsigned char)size;
2067n/a len = 2;
2068n/a }
2069n/a else if ((size_t)size <= 0xffffffffUL) {
2070n/a header[0] = BINBYTES;
2071n/a header[1] = (unsigned char)(size & 0xff);
2072n/a header[2] = (unsigned char)((size >> 8) & 0xff);
2073n/a header[3] = (unsigned char)((size >> 16) & 0xff);
2074n/a header[4] = (unsigned char)((size >> 24) & 0xff);
2075n/a len = 5;
2076n/a }
2077n/a else if (self->proto >= 4) {
2078n/a header[0] = BINBYTES8;
2079n/a _write_size64(header + 1, size);
2080n/a len = 9;
2081n/a }
2082n/a else {
2083n/a PyErr_SetString(PyExc_OverflowError,
2084n/a "cannot serialize a bytes object larger than 4 GiB");
2085n/a return -1; /* string too large */
2086n/a }
2087n/a
2088n/a if (_Pickler_Write(self, header, len) < 0)
2089n/a return -1;
2090n/a
2091n/a if (_Pickler_Write(self, PyBytes_AS_STRING(obj), size) < 0)
2092n/a return -1;
2093n/a
2094n/a if (memo_put(self, obj) < 0)
2095n/a return -1;
2096n/a
2097n/a return 0;
2098n/a }
2099n/a}
2100n/a
2101n/a/* A copy of PyUnicode_EncodeRawUnicodeEscape() that also translates
2102n/a backslash and newline characters to \uXXXX escapes. */
2103n/astatic PyObject *
2104n/araw_unicode_escape(PyObject *obj)
2105n/a{
2106n/a char *p;
2107n/a Py_ssize_t i, size;
2108n/a void *data;
2109n/a unsigned int kind;
2110n/a _PyBytesWriter writer;
2111n/a
2112n/a if (PyUnicode_READY(obj))
2113n/a return NULL;
2114n/a
2115n/a _PyBytesWriter_Init(&writer);
2116n/a
2117n/a size = PyUnicode_GET_LENGTH(obj);
2118n/a data = PyUnicode_DATA(obj);
2119n/a kind = PyUnicode_KIND(obj);
2120n/a
2121n/a p = _PyBytesWriter_Alloc(&writer, size);
2122n/a if (p == NULL)
2123n/a goto error;
2124n/a writer.overallocate = 1;
2125n/a
2126n/a for (i=0; i < size; i++) {
2127n/a Py_UCS4 ch = PyUnicode_READ(kind, data, i);
2128n/a /* Map 32-bit characters to '\Uxxxxxxxx' */
2129n/a if (ch >= 0x10000) {
2130n/a /* -1: subtract 1 preallocated byte */
2131n/a p = _PyBytesWriter_Prepare(&writer, p, 10-1);
2132n/a if (p == NULL)
2133n/a goto error;
2134n/a
2135n/a *p++ = '\\';
2136n/a *p++ = 'U';
2137n/a *p++ = Py_hexdigits[(ch >> 28) & 0xf];
2138n/a *p++ = Py_hexdigits[(ch >> 24) & 0xf];
2139n/a *p++ = Py_hexdigits[(ch >> 20) & 0xf];
2140n/a *p++ = Py_hexdigits[(ch >> 16) & 0xf];
2141n/a *p++ = Py_hexdigits[(ch >> 12) & 0xf];
2142n/a *p++ = Py_hexdigits[(ch >> 8) & 0xf];
2143n/a *p++ = Py_hexdigits[(ch >> 4) & 0xf];
2144n/a *p++ = Py_hexdigits[ch & 15];
2145n/a }
2146n/a /* Map 16-bit characters, '\\' and '\n' to '\uxxxx' */
2147n/a else if (ch >= 256 || ch == '\\' || ch == '\n') {
2148n/a /* -1: subtract 1 preallocated byte */
2149n/a p = _PyBytesWriter_Prepare(&writer, p, 6-1);
2150n/a if (p == NULL)
2151n/a goto error;
2152n/a
2153n/a *p++ = '\\';
2154n/a *p++ = 'u';
2155n/a *p++ = Py_hexdigits[(ch >> 12) & 0xf];
2156n/a *p++ = Py_hexdigits[(ch >> 8) & 0xf];
2157n/a *p++ = Py_hexdigits[(ch >> 4) & 0xf];
2158n/a *p++ = Py_hexdigits[ch & 15];
2159n/a }
2160n/a /* Copy everything else as-is */
2161n/a else
2162n/a *p++ = (char) ch;
2163n/a }
2164n/a
2165n/a return _PyBytesWriter_Finish(&writer, p);
2166n/a
2167n/aerror:
2168n/a _PyBytesWriter_Dealloc(&writer);
2169n/a return NULL;
2170n/a}
2171n/a
2172n/astatic int
2173n/awrite_utf8(PicklerObject *self, const char *data, Py_ssize_t size)
2174n/a{
2175n/a char header[9];
2176n/a Py_ssize_t len;
2177n/a
2178n/a assert(size >= 0);
2179n/a if (size <= 0xff && self->proto >= 4) {
2180n/a header[0] = SHORT_BINUNICODE;
2181n/a header[1] = (unsigned char)(size & 0xff);
2182n/a len = 2;
2183n/a }
2184n/a else if ((size_t)size <= 0xffffffffUL) {
2185n/a header[0] = BINUNICODE;
2186n/a header[1] = (unsigned char)(size & 0xff);
2187n/a header[2] = (unsigned char)((size >> 8) & 0xff);
2188n/a header[3] = (unsigned char)((size >> 16) & 0xff);
2189n/a header[4] = (unsigned char)((size >> 24) & 0xff);
2190n/a len = 5;
2191n/a }
2192n/a else if (self->proto >= 4) {
2193n/a header[0] = BINUNICODE8;
2194n/a _write_size64(header + 1, size);
2195n/a len = 9;
2196n/a }
2197n/a else {
2198n/a PyErr_SetString(PyExc_OverflowError,
2199n/a "cannot serialize a string larger than 4GiB");
2200n/a return -1;
2201n/a }
2202n/a
2203n/a if (_Pickler_Write(self, header, len) < 0)
2204n/a return -1;
2205n/a if (_Pickler_Write(self, data, size) < 0)
2206n/a return -1;
2207n/a
2208n/a return 0;
2209n/a}
2210n/a
2211n/astatic int
2212n/awrite_unicode_binary(PicklerObject *self, PyObject *obj)
2213n/a{
2214n/a PyObject *encoded = NULL;
2215n/a Py_ssize_t size;
2216n/a const char *data;
2217n/a int r;
2218n/a
2219n/a if (PyUnicode_READY(obj))
2220n/a return -1;
2221n/a
2222n/a data = PyUnicode_AsUTF8AndSize(obj, &size);
2223n/a if (data != NULL)
2224n/a return write_utf8(self, data, size);
2225n/a
2226n/a /* Issue #8383: for strings with lone surrogates, fallback on the
2227n/a "surrogatepass" error handler. */
2228n/a PyErr_Clear();
2229n/a encoded = PyUnicode_AsEncodedString(obj, "utf-8", "surrogatepass");
2230n/a if (encoded == NULL)
2231n/a return -1;
2232n/a
2233n/a r = write_utf8(self, PyBytes_AS_STRING(encoded),
2234n/a PyBytes_GET_SIZE(encoded));
2235n/a Py_DECREF(encoded);
2236n/a return r;
2237n/a}
2238n/a
2239n/astatic int
2240n/asave_unicode(PicklerObject *self, PyObject *obj)
2241n/a{
2242n/a if (self->bin) {
2243n/a if (write_unicode_binary(self, obj) < 0)
2244n/a return -1;
2245n/a }
2246n/a else {
2247n/a PyObject *encoded;
2248n/a Py_ssize_t size;
2249n/a const char unicode_op = UNICODE;
2250n/a
2251n/a encoded = raw_unicode_escape(obj);
2252n/a if (encoded == NULL)
2253n/a return -1;
2254n/a
2255n/a if (_Pickler_Write(self, &unicode_op, 1) < 0) {
2256n/a Py_DECREF(encoded);
2257n/a return -1;
2258n/a }
2259n/a
2260n/a size = PyBytes_GET_SIZE(encoded);
2261n/a if (_Pickler_Write(self, PyBytes_AS_STRING(encoded), size) < 0) {
2262n/a Py_DECREF(encoded);
2263n/a return -1;
2264n/a }
2265n/a Py_DECREF(encoded);
2266n/a
2267n/a if (_Pickler_Write(self, "\n", 1) < 0)
2268n/a return -1;
2269n/a }
2270n/a if (memo_put(self, obj) < 0)
2271n/a return -1;
2272n/a
2273n/a return 0;
2274n/a}
2275n/a
2276n/a/* A helper for save_tuple. Push the len elements in tuple t on the stack. */
2277n/astatic int
2278n/astore_tuple_elements(PicklerObject *self, PyObject *t, Py_ssize_t len)
2279n/a{
2280n/a Py_ssize_t i;
2281n/a
2282n/a assert(PyTuple_Size(t) == len);
2283n/a
2284n/a for (i = 0; i < len; i++) {
2285n/a PyObject *element = PyTuple_GET_ITEM(t, i);
2286n/a
2287n/a if (element == NULL)
2288n/a return -1;
2289n/a if (save(self, element, 0) < 0)
2290n/a return -1;
2291n/a }
2292n/a
2293n/a return 0;
2294n/a}
2295n/a
2296n/a/* Tuples are ubiquitous in the pickle protocols, so many techniques are
2297n/a * used across protocols to minimize the space needed to pickle them.
2298n/a * Tuples are also the only builtin immutable type that can be recursive
2299n/a * (a tuple can be reached from itself), and that requires some subtle
2300n/a * magic so that it works in all cases. IOW, this is a long routine.
2301n/a */
2302n/astatic int
2303n/asave_tuple(PicklerObject *self, PyObject *obj)
2304n/a{
2305n/a Py_ssize_t len, i;
2306n/a
2307n/a const char mark_op = MARK;
2308n/a const char tuple_op = TUPLE;
2309n/a const char pop_op = POP;
2310n/a const char pop_mark_op = POP_MARK;
2311n/a const char len2opcode[] = {EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3};
2312n/a
2313n/a if ((len = PyTuple_Size(obj)) < 0)
2314n/a return -1;
2315n/a
2316n/a if (len == 0) {
2317n/a char pdata[2];
2318n/a
2319n/a if (self->proto) {
2320n/a pdata[0] = EMPTY_TUPLE;
2321n/a len = 1;
2322n/a }
2323n/a else {
2324n/a pdata[0] = MARK;
2325n/a pdata[1] = TUPLE;
2326n/a len = 2;
2327n/a }
2328n/a if (_Pickler_Write(self, pdata, len) < 0)
2329n/a return -1;
2330n/a return 0;
2331n/a }
2332n/a
2333n/a /* The tuple isn't in the memo now. If it shows up there after
2334n/a * saving the tuple elements, the tuple must be recursive, in
2335n/a * which case we'll pop everything we put on the stack, and fetch
2336n/a * its value from the memo.
2337n/a */
2338n/a if (len <= 3 && self->proto >= 2) {
2339n/a /* Use TUPLE{1,2,3} opcodes. */
2340n/a if (store_tuple_elements(self, obj, len) < 0)
2341n/a return -1;
2342n/a
2343n/a if (PyMemoTable_Get(self->memo, obj)) {
2344n/a /* pop the len elements */
2345n/a for (i = 0; i < len; i++)
2346n/a if (_Pickler_Write(self, &pop_op, 1) < 0)
2347n/a return -1;
2348n/a /* fetch from memo */
2349n/a if (memo_get(self, obj) < 0)
2350n/a return -1;
2351n/a
2352n/a return 0;
2353n/a }
2354n/a else { /* Not recursive. */
2355n/a if (_Pickler_Write(self, len2opcode + len, 1) < 0)
2356n/a return -1;
2357n/a }
2358n/a goto memoize;
2359n/a }
2360n/a
2361n/a /* proto < 2 and len > 0, or proto >= 2 and len > 3.
2362n/a * Generate MARK e1 e2 ... TUPLE
2363n/a */
2364n/a if (_Pickler_Write(self, &mark_op, 1) < 0)
2365n/a return -1;
2366n/a
2367n/a if (store_tuple_elements(self, obj, len) < 0)
2368n/a return -1;
2369n/a
2370n/a if (PyMemoTable_Get(self->memo, obj)) {
2371n/a /* pop the stack stuff we pushed */
2372n/a if (self->bin) {
2373n/a if (_Pickler_Write(self, &pop_mark_op, 1) < 0)
2374n/a return -1;
2375n/a }
2376n/a else {
2377n/a /* Note that we pop one more than len, to remove
2378n/a * the MARK too.
2379n/a */
2380n/a for (i = 0; i <= len; i++)
2381n/a if (_Pickler_Write(self, &pop_op, 1) < 0)
2382n/a return -1;
2383n/a }
2384n/a /* fetch from memo */
2385n/a if (memo_get(self, obj) < 0)
2386n/a return -1;
2387n/a
2388n/a return 0;
2389n/a }
2390n/a else { /* Not recursive. */
2391n/a if (_Pickler_Write(self, &tuple_op, 1) < 0)
2392n/a return -1;
2393n/a }
2394n/a
2395n/a memoize:
2396n/a if (memo_put(self, obj) < 0)
2397n/a return -1;
2398n/a
2399n/a return 0;
2400n/a}
2401n/a
2402n/a/* iter is an iterator giving items, and we batch up chunks of
2403n/a * MARK item item ... item APPENDS
2404n/a * opcode sequences. Calling code should have arranged to first create an
2405n/a * empty list, or list-like object, for the APPENDS to operate on.
2406n/a * Returns 0 on success, <0 on error.
2407n/a */
2408n/astatic int
2409n/abatch_list(PicklerObject *self, PyObject *iter)
2410n/a{
2411n/a PyObject *obj = NULL;
2412n/a PyObject *firstitem = NULL;
2413n/a int i, n;
2414n/a
2415n/a const char mark_op = MARK;
2416n/a const char append_op = APPEND;
2417n/a const char appends_op = APPENDS;
2418n/a
2419n/a assert(iter != NULL);
2420n/a
2421n/a /* XXX: I think this function could be made faster by avoiding the
2422n/a iterator interface and fetching objects directly from list using
2423n/a PyList_GET_ITEM.
2424n/a */
2425n/a
2426n/a if (self->proto == 0) {
2427n/a /* APPENDS isn't available; do one at a time. */
2428n/a for (;;) {
2429n/a obj = PyIter_Next(iter);
2430n/a if (obj == NULL) {
2431n/a if (PyErr_Occurred())
2432n/a return -1;
2433n/a break;
2434n/a }
2435n/a i = save(self, obj, 0);
2436n/a Py_DECREF(obj);
2437n/a if (i < 0)
2438n/a return -1;
2439n/a if (_Pickler_Write(self, &append_op, 1) < 0)
2440n/a return -1;
2441n/a }
2442n/a return 0;
2443n/a }
2444n/a
2445n/a /* proto > 0: write in batches of BATCHSIZE. */
2446n/a do {
2447n/a /* Get first item */
2448n/a firstitem = PyIter_Next(iter);
2449n/a if (firstitem == NULL) {
2450n/a if (PyErr_Occurred())
2451n/a goto error;
2452n/a
2453n/a /* nothing more to add */
2454n/a break;
2455n/a }
2456n/a
2457n/a /* Try to get a second item */
2458n/a obj = PyIter_Next(iter);
2459n/a if (obj == NULL) {
2460n/a if (PyErr_Occurred())
2461n/a goto error;
2462n/a
2463n/a /* Only one item to write */
2464n/a if (save(self, firstitem, 0) < 0)
2465n/a goto error;
2466n/a if (_Pickler_Write(self, &append_op, 1) < 0)
2467n/a goto error;
2468n/a Py_CLEAR(firstitem);
2469n/a break;
2470n/a }
2471n/a
2472n/a /* More than one item to write */
2473n/a
2474n/a /* Pump out MARK, items, APPENDS. */
2475n/a if (_Pickler_Write(self, &mark_op, 1) < 0)
2476n/a goto error;
2477n/a
2478n/a if (save(self, firstitem, 0) < 0)
2479n/a goto error;
2480n/a Py_CLEAR(firstitem);
2481n/a n = 1;
2482n/a
2483n/a /* Fetch and save up to BATCHSIZE items */
2484n/a while (obj) {
2485n/a if (save(self, obj, 0) < 0)
2486n/a goto error;
2487n/a Py_CLEAR(obj);
2488n/a n += 1;
2489n/a
2490n/a if (n == BATCHSIZE)
2491n/a break;
2492n/a
2493n/a obj = PyIter_Next(iter);
2494n/a if (obj == NULL) {
2495n/a if (PyErr_Occurred())
2496n/a goto error;
2497n/a break;
2498n/a }
2499n/a }
2500n/a
2501n/a if (_Pickler_Write(self, &appends_op, 1) < 0)
2502n/a goto error;
2503n/a
2504n/a } while (n == BATCHSIZE);
2505n/a return 0;
2506n/a
2507n/a error:
2508n/a Py_XDECREF(firstitem);
2509n/a Py_XDECREF(obj);
2510n/a return -1;
2511n/a}
2512n/a
2513n/a/* This is a variant of batch_list() above, specialized for lists (with no
2514n/a * support for list subclasses). Like batch_list(), we batch up chunks of
2515n/a * MARK item item ... item APPENDS
2516n/a * opcode sequences. Calling code should have arranged to first create an
2517n/a * empty list, or list-like object, for the APPENDS to operate on.
2518n/a * Returns 0 on success, -1 on error.
2519n/a *
2520n/a * This version is considerably faster than batch_list(), if less general.
2521n/a *
2522n/a * Note that this only works for protocols > 0.
2523n/a */
2524n/astatic int
2525n/abatch_list_exact(PicklerObject *self, PyObject *obj)
2526n/a{
2527n/a PyObject *item = NULL;
2528n/a Py_ssize_t this_batch, total;
2529n/a
2530n/a const char append_op = APPEND;
2531n/a const char appends_op = APPENDS;
2532n/a const char mark_op = MARK;
2533n/a
2534n/a assert(obj != NULL);
2535n/a assert(self->proto > 0);
2536n/a assert(PyList_CheckExact(obj));
2537n/a
2538n/a if (PyList_GET_SIZE(obj) == 1) {
2539n/a item = PyList_GET_ITEM(obj, 0);
2540n/a if (save(self, item, 0) < 0)
2541n/a return -1;
2542n/a if (_Pickler_Write(self, &append_op, 1) < 0)
2543n/a return -1;
2544n/a return 0;
2545n/a }
2546n/a
2547n/a /* Write in batches of BATCHSIZE. */
2548n/a total = 0;
2549n/a do {
2550n/a this_batch = 0;
2551n/a if (_Pickler_Write(self, &mark_op, 1) < 0)
2552n/a return -1;
2553n/a while (total < PyList_GET_SIZE(obj)) {
2554n/a item = PyList_GET_ITEM(obj, total);
2555n/a if (save(self, item, 0) < 0)
2556n/a return -1;
2557n/a total++;
2558n/a if (++this_batch == BATCHSIZE)
2559n/a break;
2560n/a }
2561n/a if (_Pickler_Write(self, &appends_op, 1) < 0)
2562n/a return -1;
2563n/a
2564n/a } while (total < PyList_GET_SIZE(obj));
2565n/a
2566n/a return 0;
2567n/a}
2568n/a
2569n/astatic int
2570n/asave_list(PicklerObject *self, PyObject *obj)
2571n/a{
2572n/a char header[3];
2573n/a Py_ssize_t len;
2574n/a int status = 0;
2575n/a
2576n/a if (self->fast && !fast_save_enter(self, obj))
2577n/a goto error;
2578n/a
2579n/a /* Create an empty list. */
2580n/a if (self->bin) {
2581n/a header[0] = EMPTY_LIST;
2582n/a len = 1;
2583n/a }
2584n/a else {
2585n/a header[0] = MARK;
2586n/a header[1] = LIST;
2587n/a len = 2;
2588n/a }
2589n/a
2590n/a if (_Pickler_Write(self, header, len) < 0)
2591n/a goto error;
2592n/a
2593n/a /* Get list length, and bow out early if empty. */
2594n/a if ((len = PyList_Size(obj)) < 0)
2595n/a goto error;
2596n/a
2597n/a if (memo_put(self, obj) < 0)
2598n/a goto error;
2599n/a
2600n/a if (len != 0) {
2601n/a /* Materialize the list elements. */
2602n/a if (PyList_CheckExact(obj) && self->proto > 0) {
2603n/a if (Py_EnterRecursiveCall(" while pickling an object"))
2604n/a goto error;
2605n/a status = batch_list_exact(self, obj);
2606n/a Py_LeaveRecursiveCall();
2607n/a } else {
2608n/a PyObject *iter = PyObject_GetIter(obj);
2609n/a if (iter == NULL)
2610n/a goto error;
2611n/a
2612n/a if (Py_EnterRecursiveCall(" while pickling an object")) {
2613n/a Py_DECREF(iter);
2614n/a goto error;
2615n/a }
2616n/a status = batch_list(self, iter);
2617n/a Py_LeaveRecursiveCall();
2618n/a Py_DECREF(iter);
2619n/a }
2620n/a }
2621n/a if (0) {
2622n/a error:
2623n/a status = -1;
2624n/a }
2625n/a
2626n/a if (self->fast && !fast_save_leave(self, obj))
2627n/a status = -1;
2628n/a
2629n/a return status;
2630n/a}
2631n/a
2632n/a/* iter is an iterator giving (key, value) pairs, and we batch up chunks of
2633n/a * MARK key value ... key value SETITEMS
2634n/a * opcode sequences. Calling code should have arranged to first create an
2635n/a * empty dict, or dict-like object, for the SETITEMS to operate on.
2636n/a * Returns 0 on success, <0 on error.
2637n/a *
2638n/a * This is very much like batch_list(). The difference between saving
2639n/a * elements directly, and picking apart two-tuples, is so long-winded at
2640n/a * the C level, though, that attempts to combine these routines were too
2641n/a * ugly to bear.
2642n/a */
2643n/astatic int
2644n/abatch_dict(PicklerObject *self, PyObject *iter)
2645n/a{
2646n/a PyObject *obj = NULL;
2647n/a PyObject *firstitem = NULL;
2648n/a int i, n;
2649n/a
2650n/a const char mark_op = MARK;
2651n/a const char setitem_op = SETITEM;
2652n/a const char setitems_op = SETITEMS;
2653n/a
2654n/a assert(iter != NULL);
2655n/a
2656n/a if (self->proto == 0) {
2657n/a /* SETITEMS isn't available; do one at a time. */
2658n/a for (;;) {
2659n/a obj = PyIter_Next(iter);
2660n/a if (obj == NULL) {
2661n/a if (PyErr_Occurred())
2662n/a return -1;
2663n/a break;
2664n/a }
2665n/a if (!PyTuple_Check(obj) || PyTuple_Size(obj) != 2) {
2666n/a PyErr_SetString(PyExc_TypeError, "dict items "
2667n/a "iterator must return 2-tuples");
2668n/a return -1;
2669n/a }
2670n/a i = save(self, PyTuple_GET_ITEM(obj, 0), 0);
2671n/a if (i >= 0)
2672n/a i = save(self, PyTuple_GET_ITEM(obj, 1), 0);
2673n/a Py_DECREF(obj);
2674n/a if (i < 0)
2675n/a return -1;
2676n/a if (_Pickler_Write(self, &setitem_op, 1) < 0)
2677n/a return -1;
2678n/a }
2679n/a return 0;
2680n/a }
2681n/a
2682n/a /* proto > 0: write in batches of BATCHSIZE. */
2683n/a do {
2684n/a /* Get first item */
2685n/a firstitem = PyIter_Next(iter);
2686n/a if (firstitem == NULL) {
2687n/a if (PyErr_Occurred())
2688n/a goto error;
2689n/a
2690n/a /* nothing more to add */
2691n/a break;
2692n/a }
2693n/a if (!PyTuple_Check(firstitem) || PyTuple_Size(firstitem) != 2) {
2694n/a PyErr_SetString(PyExc_TypeError, "dict items "
2695n/a "iterator must return 2-tuples");
2696n/a goto error;
2697n/a }
2698n/a
2699n/a /* Try to get a second item */
2700n/a obj = PyIter_Next(iter);
2701n/a if (obj == NULL) {
2702n/a if (PyErr_Occurred())
2703n/a goto error;
2704n/a
2705n/a /* Only one item to write */
2706n/a if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0)
2707n/a goto error;
2708n/a if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0)
2709n/a goto error;
2710n/a if (_Pickler_Write(self, &setitem_op, 1) < 0)
2711n/a goto error;
2712n/a Py_CLEAR(firstitem);
2713n/a break;
2714n/a }
2715n/a
2716n/a /* More than one item to write */
2717n/a
2718n/a /* Pump out MARK, items, SETITEMS. */
2719n/a if (_Pickler_Write(self, &mark_op, 1) < 0)
2720n/a goto error;
2721n/a
2722n/a if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0)
2723n/a goto error;
2724n/a if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0)
2725n/a goto error;
2726n/a Py_CLEAR(firstitem);
2727n/a n = 1;
2728n/a
2729n/a /* Fetch and save up to BATCHSIZE items */
2730n/a while (obj) {
2731n/a if (!PyTuple_Check(obj) || PyTuple_Size(obj) != 2) {
2732n/a PyErr_SetString(PyExc_TypeError, "dict items "
2733n/a "iterator must return 2-tuples");
2734n/a goto error;
2735n/a }
2736n/a if (save(self, PyTuple_GET_ITEM(obj, 0), 0) < 0 ||
2737n/a save(self, PyTuple_GET_ITEM(obj, 1), 0) < 0)
2738n/a goto error;
2739n/a Py_CLEAR(obj);
2740n/a n += 1;
2741n/a
2742n/a if (n == BATCHSIZE)
2743n/a break;
2744n/a
2745n/a obj = PyIter_Next(iter);
2746n/a if (obj == NULL) {
2747n/a if (PyErr_Occurred())
2748n/a goto error;
2749n/a break;
2750n/a }
2751n/a }
2752n/a
2753n/a if (_Pickler_Write(self, &setitems_op, 1) < 0)
2754n/a goto error;
2755n/a
2756n/a } while (n == BATCHSIZE);
2757n/a return 0;
2758n/a
2759n/a error:
2760n/a Py_XDECREF(firstitem);
2761n/a Py_XDECREF(obj);
2762n/a return -1;
2763n/a}
2764n/a
2765n/a/* This is a variant of batch_dict() above that specializes for dicts, with no
2766n/a * support for dict subclasses. Like batch_dict(), we batch up chunks of
2767n/a * MARK key value ... key value SETITEMS
2768n/a * opcode sequences. Calling code should have arranged to first create an
2769n/a * empty dict, or dict-like object, for the SETITEMS to operate on.
2770n/a * Returns 0 on success, -1 on error.
2771n/a *
2772n/a * Note that this currently doesn't work for protocol 0.
2773n/a */
2774n/astatic int
2775n/abatch_dict_exact(PicklerObject *self, PyObject *obj)
2776n/a{
2777n/a PyObject *key = NULL, *value = NULL;
2778n/a int i;
2779n/a Py_ssize_t dict_size, ppos = 0;
2780n/a
2781n/a const char mark_op = MARK;
2782n/a const char setitem_op = SETITEM;
2783n/a const char setitems_op = SETITEMS;
2784n/a
2785n/a assert(obj != NULL && PyDict_CheckExact(obj));
2786n/a assert(self->proto > 0);
2787n/a
2788n/a dict_size = PyDict_GET_SIZE(obj);
2789n/a
2790n/a /* Special-case len(d) == 1 to save space. */
2791n/a if (dict_size == 1) {
2792n/a PyDict_Next(obj, &ppos, &key, &value);
2793n/a if (save(self, key, 0) < 0)
2794n/a return -1;
2795n/a if (save(self, value, 0) < 0)
2796n/a return -1;
2797n/a if (_Pickler_Write(self, &setitem_op, 1) < 0)
2798n/a return -1;
2799n/a return 0;
2800n/a }
2801n/a
2802n/a /* Write in batches of BATCHSIZE. */
2803n/a do {
2804n/a i = 0;
2805n/a if (_Pickler_Write(self, &mark_op, 1) < 0)
2806n/a return -1;
2807n/a while (PyDict_Next(obj, &ppos, &key, &value)) {
2808n/a if (save(self, key, 0) < 0)
2809n/a return -1;
2810n/a if (save(self, value, 0) < 0)
2811n/a return -1;
2812n/a if (++i == BATCHSIZE)
2813n/a break;
2814n/a }
2815n/a if (_Pickler_Write(self, &setitems_op, 1) < 0)
2816n/a return -1;
2817n/a if (PyDict_GET_SIZE(obj) != dict_size) {
2818n/a PyErr_Format(
2819n/a PyExc_RuntimeError,
2820n/a "dictionary changed size during iteration");
2821n/a return -1;
2822n/a }
2823n/a
2824n/a } while (i == BATCHSIZE);
2825n/a return 0;
2826n/a}
2827n/a
2828n/astatic int
2829n/asave_dict(PicklerObject *self, PyObject *obj)
2830n/a{
2831n/a PyObject *items, *iter;
2832n/a char header[3];
2833n/a Py_ssize_t len;
2834n/a int status = 0;
2835n/a assert(PyDict_Check(obj));
2836n/a
2837n/a if (self->fast && !fast_save_enter(self, obj))
2838n/a goto error;
2839n/a
2840n/a /* Create an empty dict. */
2841n/a if (self->bin) {
2842n/a header[0] = EMPTY_DICT;
2843n/a len = 1;
2844n/a }
2845n/a else {
2846n/a header[0] = MARK;
2847n/a header[1] = DICT;
2848n/a len = 2;
2849n/a }
2850n/a
2851n/a if (_Pickler_Write(self, header, len) < 0)
2852n/a goto error;
2853n/a
2854n/a if (memo_put(self, obj) < 0)
2855n/a goto error;
2856n/a
2857n/a if (PyDict_GET_SIZE(obj)) {
2858n/a /* Save the dict items. */
2859n/a if (PyDict_CheckExact(obj) && self->proto > 0) {
2860n/a /* We can take certain shortcuts if we know this is a dict and
2861n/a not a dict subclass. */
2862n/a if (Py_EnterRecursiveCall(" while pickling an object"))
2863n/a goto error;
2864n/a status = batch_dict_exact(self, obj);
2865n/a Py_LeaveRecursiveCall();
2866n/a } else {
2867n/a _Py_IDENTIFIER(items);
2868n/a
2869n/a items = _PyObject_CallMethodId(obj, &PyId_items, NULL);
2870n/a if (items == NULL)
2871n/a goto error;
2872n/a iter = PyObject_GetIter(items);
2873n/a Py_DECREF(items);
2874n/a if (iter == NULL)
2875n/a goto error;
2876n/a if (Py_EnterRecursiveCall(" while pickling an object")) {
2877n/a Py_DECREF(iter);
2878n/a goto error;
2879n/a }
2880n/a status = batch_dict(self, iter);
2881n/a Py_LeaveRecursiveCall();
2882n/a Py_DECREF(iter);
2883n/a }
2884n/a }
2885n/a
2886n/a if (0) {
2887n/a error:
2888n/a status = -1;
2889n/a }
2890n/a
2891n/a if (self->fast && !fast_save_leave(self, obj))
2892n/a status = -1;
2893n/a
2894n/a return status;
2895n/a}
2896n/a
2897n/astatic int
2898n/asave_set(PicklerObject *self, PyObject *obj)
2899n/a{
2900n/a PyObject *item;
2901n/a int i;
2902n/a Py_ssize_t set_size, ppos = 0;
2903n/a Py_hash_t hash;
2904n/a
2905n/a const char empty_set_op = EMPTY_SET;
2906n/a const char mark_op = MARK;
2907n/a const char additems_op = ADDITEMS;
2908n/a
2909n/a if (self->proto < 4) {
2910n/a PyObject *items;
2911n/a PyObject *reduce_value;
2912n/a int status;
2913n/a
2914n/a items = PySequence_List(obj);
2915n/a if (items == NULL) {
2916n/a return -1;
2917n/a }
2918n/a reduce_value = Py_BuildValue("(O(O))", (PyObject*)&PySet_Type, items);
2919n/a Py_DECREF(items);
2920n/a if (reduce_value == NULL) {
2921n/a return -1;
2922n/a }
2923n/a /* save_reduce() will memoize the object automatically. */
2924n/a status = save_reduce(self, reduce_value, obj);
2925n/a Py_DECREF(reduce_value);
2926n/a return status;
2927n/a }
2928n/a
2929n/a if (_Pickler_Write(self, &empty_set_op, 1) < 0)
2930n/a return -1;
2931n/a
2932n/a if (memo_put(self, obj) < 0)
2933n/a return -1;
2934n/a
2935n/a set_size = PySet_GET_SIZE(obj);
2936n/a if (set_size == 0)
2937n/a return 0; /* nothing to do */
2938n/a
2939n/a /* Write in batches of BATCHSIZE. */
2940n/a do {
2941n/a i = 0;
2942n/a if (_Pickler_Write(self, &mark_op, 1) < 0)
2943n/a return -1;
2944n/a while (_PySet_NextEntry(obj, &ppos, &item, &hash)) {
2945n/a if (save(self, item, 0) < 0)
2946n/a return -1;
2947n/a if (++i == BATCHSIZE)
2948n/a break;
2949n/a }
2950n/a if (_Pickler_Write(self, &additems_op, 1) < 0)
2951n/a return -1;
2952n/a if (PySet_GET_SIZE(obj) != set_size) {
2953n/a PyErr_Format(
2954n/a PyExc_RuntimeError,
2955n/a "set changed size during iteration");
2956n/a return -1;
2957n/a }
2958n/a } while (i == BATCHSIZE);
2959n/a
2960n/a return 0;
2961n/a}
2962n/a
2963n/astatic int
2964n/asave_frozenset(PicklerObject *self, PyObject *obj)
2965n/a{
2966n/a PyObject *iter;
2967n/a
2968n/a const char mark_op = MARK;
2969n/a const char frozenset_op = FROZENSET;
2970n/a
2971n/a if (self->fast && !fast_save_enter(self, obj))
2972n/a return -1;
2973n/a
2974n/a if (self->proto < 4) {
2975n/a PyObject *items;
2976n/a PyObject *reduce_value;
2977n/a int status;
2978n/a
2979n/a items = PySequence_List(obj);
2980n/a if (items == NULL) {
2981n/a return -1;
2982n/a }
2983n/a reduce_value = Py_BuildValue("(O(O))", (PyObject*)&PyFrozenSet_Type,
2984n/a items);
2985n/a Py_DECREF(items);
2986n/a if (reduce_value == NULL) {
2987n/a return -1;
2988n/a }
2989n/a /* save_reduce() will memoize the object automatically. */
2990n/a status = save_reduce(self, reduce_value, obj);
2991n/a Py_DECREF(reduce_value);
2992n/a return status;
2993n/a }
2994n/a
2995n/a if (_Pickler_Write(self, &mark_op, 1) < 0)
2996n/a return -1;
2997n/a
2998n/a iter = PyObject_GetIter(obj);
2999n/a if (iter == NULL) {
3000n/a return -1;
3001n/a }
3002n/a for (;;) {
3003n/a PyObject *item;
3004n/a
3005n/a item = PyIter_Next(iter);
3006n/a if (item == NULL) {
3007n/a if (PyErr_Occurred()) {
3008n/a Py_DECREF(iter);
3009n/a return -1;
3010n/a }
3011n/a break;
3012n/a }
3013n/a if (save(self, item, 0) < 0) {
3014n/a Py_DECREF(item);
3015n/a Py_DECREF(iter);
3016n/a return -1;
3017n/a }
3018n/a Py_DECREF(item);
3019n/a }
3020n/a Py_DECREF(iter);
3021n/a
3022n/a /* If the object is already in the memo, this means it is
3023n/a recursive. In this case, throw away everything we put on the
3024n/a stack, and fetch the object back from the memo. */
3025n/a if (PyMemoTable_Get(self->memo, obj)) {
3026n/a const char pop_mark_op = POP_MARK;
3027n/a
3028n/a if (_Pickler_Write(self, &pop_mark_op, 1) < 0)
3029n/a return -1;
3030n/a if (memo_get(self, obj) < 0)
3031n/a return -1;
3032n/a return 0;
3033n/a }
3034n/a
3035n/a if (_Pickler_Write(self, &frozenset_op, 1) < 0)
3036n/a return -1;
3037n/a if (memo_put(self, obj) < 0)
3038n/a return -1;
3039n/a
3040n/a return 0;
3041n/a}
3042n/a
3043n/astatic int
3044n/afix_imports(PyObject **module_name, PyObject **global_name)
3045n/a{
3046n/a PyObject *key;
3047n/a PyObject *item;
3048n/a PickleState *st = _Pickle_GetGlobalState();
3049n/a
3050n/a key = PyTuple_Pack(2, *module_name, *global_name);
3051n/a if (key == NULL)
3052n/a return -1;
3053n/a item = PyDict_GetItemWithError(st->name_mapping_3to2, key);
3054n/a Py_DECREF(key);
3055n/a if (item) {
3056n/a PyObject *fixed_module_name;
3057n/a PyObject *fixed_global_name;
3058n/a
3059n/a if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) {
3060n/a PyErr_Format(PyExc_RuntimeError,
3061n/a "_compat_pickle.REVERSE_NAME_MAPPING values "
3062n/a "should be 2-tuples, not %.200s",
3063n/a Py_TYPE(item)->tp_name);
3064n/a return -1;
3065n/a }
3066n/a fixed_module_name = PyTuple_GET_ITEM(item, 0);
3067n/a fixed_global_name = PyTuple_GET_ITEM(item, 1);
3068n/a if (!PyUnicode_Check(fixed_module_name) ||
3069n/a !PyUnicode_Check(fixed_global_name)) {
3070n/a PyErr_Format(PyExc_RuntimeError,
3071n/a "_compat_pickle.REVERSE_NAME_MAPPING values "
3072n/a "should be pairs of str, not (%.200s, %.200s)",
3073n/a Py_TYPE(fixed_module_name)->tp_name,
3074n/a Py_TYPE(fixed_global_name)->tp_name);
3075n/a return -1;
3076n/a }
3077n/a
3078n/a Py_CLEAR(*module_name);
3079n/a Py_CLEAR(*global_name);
3080n/a Py_INCREF(fixed_module_name);
3081n/a Py_INCREF(fixed_global_name);
3082n/a *module_name = fixed_module_name;
3083n/a *global_name = fixed_global_name;
3084n/a return 0;
3085n/a }
3086n/a else if (PyErr_Occurred()) {
3087n/a return -1;
3088n/a }
3089n/a
3090n/a item = PyDict_GetItemWithError(st->import_mapping_3to2, *module_name);
3091n/a if (item) {
3092n/a if (!PyUnicode_Check(item)) {
3093n/a PyErr_Format(PyExc_RuntimeError,
3094n/a "_compat_pickle.REVERSE_IMPORT_MAPPING values "
3095n/a "should be strings, not %.200s",
3096n/a Py_TYPE(item)->tp_name);
3097n/a return -1;
3098n/a }
3099n/a Py_INCREF(item);
3100n/a Py_XSETREF(*module_name, item);
3101n/a }
3102n/a else if (PyErr_Occurred()) {
3103n/a return -1;
3104n/a }
3105n/a
3106n/a return 0;
3107n/a}
3108n/a
3109n/astatic int
3110n/asave_global(PicklerObject *self, PyObject *obj, PyObject *name)
3111n/a{
3112n/a PyObject *global_name = NULL;
3113n/a PyObject *module_name = NULL;
3114n/a PyObject *module = NULL;
3115n/a PyObject *parent = NULL;
3116n/a PyObject *dotted_path = NULL;
3117n/a PyObject *lastname = NULL;
3118n/a PyObject *cls;
3119n/a PickleState *st = _Pickle_GetGlobalState();
3120n/a int status = 0;
3121n/a _Py_IDENTIFIER(__name__);
3122n/a _Py_IDENTIFIER(__qualname__);
3123n/a
3124n/a const char global_op = GLOBAL;
3125n/a
3126n/a if (name) {
3127n/a Py_INCREF(name);
3128n/a global_name = name;
3129n/a }
3130n/a else {
3131n/a global_name = _PyObject_GetAttrId(obj, &PyId___qualname__);
3132n/a if (global_name == NULL) {
3133n/a if (!PyErr_ExceptionMatches(PyExc_AttributeError))
3134n/a goto error;
3135n/a PyErr_Clear();
3136n/a }
3137n/a if (global_name == NULL) {
3138n/a global_name = _PyObject_GetAttrId(obj, &PyId___name__);
3139n/a if (global_name == NULL)
3140n/a goto error;
3141n/a }
3142n/a }
3143n/a
3144n/a dotted_path = get_dotted_path(module, global_name);
3145n/a if (dotted_path == NULL)
3146n/a goto error;
3147n/a module_name = whichmodule(obj, dotted_path);
3148n/a if (module_name == NULL)
3149n/a goto error;
3150n/a
3151n/a /* XXX: Change to use the import C API directly with level=0 to disallow
3152n/a relative imports.
3153n/a
3154n/a XXX: PyImport_ImportModuleLevel could be used. However, this bypasses
3155n/a builtins.__import__. Therefore, _pickle, unlike pickle.py, will ignore
3156n/a custom import functions (IMHO, this would be a nice security
3157n/a feature). The import C API would need to be extended to support the
3158n/a extra parameters of __import__ to fix that. */
3159n/a module = PyImport_Import(module_name);
3160n/a if (module == NULL) {
3161n/a PyErr_Format(st->PicklingError,
3162n/a "Can't pickle %R: import of module %R failed",
3163n/a obj, module_name);
3164n/a goto error;
3165n/a }
3166n/a lastname = PyList_GET_ITEM(dotted_path, PyList_GET_SIZE(dotted_path)-1);
3167n/a Py_INCREF(lastname);
3168n/a cls = get_deep_attribute(module, dotted_path, &parent);
3169n/a Py_CLEAR(dotted_path);
3170n/a if (cls == NULL) {
3171n/a PyErr_Format(st->PicklingError,
3172n/a "Can't pickle %R: attribute lookup %S on %S failed",
3173n/a obj, global_name, module_name);
3174n/a goto error;
3175n/a }
3176n/a if (cls != obj) {
3177n/a Py_DECREF(cls);
3178n/a PyErr_Format(st->PicklingError,
3179n/a "Can't pickle %R: it's not the same object as %S.%S",
3180n/a obj, module_name, global_name);
3181n/a goto error;
3182n/a }
3183n/a Py_DECREF(cls);
3184n/a
3185n/a if (self->proto >= 2) {
3186n/a /* See whether this is in the extension registry, and if
3187n/a * so generate an EXT opcode.
3188n/a */
3189n/a PyObject *extension_key;
3190n/a PyObject *code_obj; /* extension code as Python object */
3191n/a long code; /* extension code as C value */
3192n/a char pdata[5];
3193n/a Py_ssize_t n;
3194n/a
3195n/a extension_key = PyTuple_Pack(2, module_name, global_name);
3196n/a if (extension_key == NULL) {
3197n/a goto error;
3198n/a }
3199n/a code_obj = PyDict_GetItemWithError(st->extension_registry,
3200n/a extension_key);
3201n/a Py_DECREF(extension_key);
3202n/a /* The object is not registered in the extension registry.
3203n/a This is the most likely code path. */
3204n/a if (code_obj == NULL) {
3205n/a if (PyErr_Occurred()) {
3206n/a goto error;
3207n/a }
3208n/a goto gen_global;
3209n/a }
3210n/a
3211n/a /* XXX: pickle.py doesn't check neither the type, nor the range
3212n/a of the value returned by the extension_registry. It should for
3213n/a consistency. */
3214n/a
3215n/a /* Verify code_obj has the right type and value. */
3216n/a if (!PyLong_Check(code_obj)) {
3217n/a PyErr_Format(st->PicklingError,
3218n/a "Can't pickle %R: extension code %R isn't an integer",
3219n/a obj, code_obj);
3220n/a goto error;
3221n/a }
3222n/a code = PyLong_AS_LONG(code_obj);
3223n/a if (code <= 0 || code > 0x7fffffffL) {
3224n/a if (!PyErr_Occurred())
3225n/a PyErr_Format(st->PicklingError, "Can't pickle %R: extension "
3226n/a "code %ld is out of range", obj, code);
3227n/a goto error;
3228n/a }
3229n/a
3230n/a /* Generate an EXT opcode. */
3231n/a if (code <= 0xff) {
3232n/a pdata[0] = EXT1;
3233n/a pdata[1] = (unsigned char)code;
3234n/a n = 2;
3235n/a }
3236n/a else if (code <= 0xffff) {
3237n/a pdata[0] = EXT2;
3238n/a pdata[1] = (unsigned char)(code & 0xff);
3239n/a pdata[2] = (unsigned char)((code >> 8) & 0xff);
3240n/a n = 3;
3241n/a }
3242n/a else {
3243n/a pdata[0] = EXT4;
3244n/a pdata[1] = (unsigned char)(code & 0xff);
3245n/a pdata[2] = (unsigned char)((code >> 8) & 0xff);
3246n/a pdata[3] = (unsigned char)((code >> 16) & 0xff);
3247n/a pdata[4] = (unsigned char)((code >> 24) & 0xff);
3248n/a n = 5;
3249n/a }
3250n/a
3251n/a if (_Pickler_Write(self, pdata, n) < 0)
3252n/a goto error;
3253n/a }
3254n/a else {
3255n/a gen_global:
3256n/a if (parent == module) {
3257n/a Py_INCREF(lastname);
3258n/a Py_DECREF(global_name);
3259n/a global_name = lastname;
3260n/a }
3261n/a if (self->proto >= 4) {
3262n/a const char stack_global_op = STACK_GLOBAL;
3263n/a
3264n/a if (save(self, module_name, 0) < 0)
3265n/a goto error;
3266n/a if (save(self, global_name, 0) < 0)
3267n/a goto error;
3268n/a
3269n/a if (_Pickler_Write(self, &stack_global_op, 1) < 0)
3270n/a goto error;
3271n/a }
3272n/a else if (parent != module) {
3273n/a PickleState *st = _Pickle_GetGlobalState();
3274n/a PyObject *reduce_value = Py_BuildValue("(O(OO))",
3275n/a st->getattr, parent, lastname);
3276n/a status = save_reduce(self, reduce_value, NULL);
3277n/a Py_DECREF(reduce_value);
3278n/a if (status < 0)
3279n/a goto error;
3280n/a }
3281n/a else {
3282n/a /* Generate a normal global opcode if we are using a pickle
3283n/a protocol < 4, or if the object is not registered in the
3284n/a extension registry. */
3285n/a PyObject *encoded;
3286n/a PyObject *(*unicode_encoder)(PyObject *);
3287n/a
3288n/a if (_Pickler_Write(self, &global_op, 1) < 0)
3289n/a goto error;
3290n/a
3291n/a /* For protocol < 3 and if the user didn't request against doing
3292n/a so, we convert module names to the old 2.x module names. */
3293n/a if (self->proto < 3 && self->fix_imports) {
3294n/a if (fix_imports(&module_name, &global_name) < 0) {
3295n/a goto error;
3296n/a }
3297n/a }
3298n/a
3299n/a /* Since Python 3.0 now supports non-ASCII identifiers, we encode
3300n/a both the module name and the global name using UTF-8. We do so
3301n/a only when we are using the pickle protocol newer than version
3302n/a 3. This is to ensure compatibility with older Unpickler running
3303n/a on Python 2.x. */
3304n/a if (self->proto == 3) {
3305n/a unicode_encoder = PyUnicode_AsUTF8String;
3306n/a }
3307n/a else {
3308n/a unicode_encoder = PyUnicode_AsASCIIString;
3309n/a }
3310n/a encoded = unicode_encoder(module_name);
3311n/a if (encoded == NULL) {
3312n/a if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError))
3313n/a PyErr_Format(st->PicklingError,
3314n/a "can't pickle module identifier '%S' using "
3315n/a "pickle protocol %i",
3316n/a module_name, self->proto);
3317n/a goto error;
3318n/a }
3319n/a if (_Pickler_Write(self, PyBytes_AS_STRING(encoded),
3320n/a PyBytes_GET_SIZE(encoded)) < 0) {
3321n/a Py_DECREF(encoded);
3322n/a goto error;
3323n/a }
3324n/a Py_DECREF(encoded);
3325n/a if(_Pickler_Write(self, "\n", 1) < 0)
3326n/a goto error;
3327n/a
3328n/a /* Save the name of the module. */
3329n/a encoded = unicode_encoder(global_name);
3330n/a if (encoded == NULL) {
3331n/a if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError))
3332n/a PyErr_Format(st->PicklingError,
3333n/a "can't pickle global identifier '%S' using "
3334n/a "pickle protocol %i",
3335n/a global_name, self->proto);
3336n/a goto error;
3337n/a }
3338n/a if (_Pickler_Write(self, PyBytes_AS_STRING(encoded),
3339n/a PyBytes_GET_SIZE(encoded)) < 0) {
3340n/a Py_DECREF(encoded);
3341n/a goto error;
3342n/a }
3343n/a Py_DECREF(encoded);
3344n/a if (_Pickler_Write(self, "\n", 1) < 0)
3345n/a goto error;
3346n/a }
3347n/a /* Memoize the object. */
3348n/a if (memo_put(self, obj) < 0)
3349n/a goto error;
3350n/a }
3351n/a
3352n/a if (0) {
3353n/a error:
3354n/a status = -1;
3355n/a }
3356n/a Py_XDECREF(module_name);
3357n/a Py_XDECREF(global_name);
3358n/a Py_XDECREF(module);
3359n/a Py_XDECREF(parent);
3360n/a Py_XDECREF(dotted_path);
3361n/a Py_XDECREF(lastname);
3362n/a
3363n/a return status;
3364n/a}
3365n/a
3366n/astatic int
3367n/asave_singleton_type(PicklerObject *self, PyObject *obj, PyObject *singleton)
3368n/a{
3369n/a PyObject *reduce_value;
3370n/a int status;
3371n/a
3372n/a reduce_value = Py_BuildValue("O(O)", &PyType_Type, singleton);
3373n/a if (reduce_value == NULL) {
3374n/a return -1;
3375n/a }
3376n/a status = save_reduce(self, reduce_value, obj);
3377n/a Py_DECREF(reduce_value);
3378n/a return status;
3379n/a}
3380n/a
3381n/astatic int
3382n/asave_type(PicklerObject *self, PyObject *obj)
3383n/a{
3384n/a if (obj == (PyObject *)&_PyNone_Type) {
3385n/a return save_singleton_type(self, obj, Py_None);
3386n/a }
3387n/a else if (obj == (PyObject *)&PyEllipsis_Type) {
3388n/a return save_singleton_type(self, obj, Py_Ellipsis);
3389n/a }
3390n/a else if (obj == (PyObject *)&_PyNotImplemented_Type) {
3391n/a return save_singleton_type(self, obj, Py_NotImplemented);
3392n/a }
3393n/a return save_global(self, obj, NULL);
3394n/a}
3395n/a
3396n/astatic int
3397n/asave_pers(PicklerObject *self, PyObject *obj, PyObject *func)
3398n/a{
3399n/a PyObject *pid = NULL;
3400n/a int status = 0;
3401n/a
3402n/a const char persid_op = PERSID;
3403n/a const char binpersid_op = BINPERSID;
3404n/a
3405n/a Py_INCREF(obj);
3406n/a pid = _Pickle_FastCall(func, obj);
3407n/a if (pid == NULL)
3408n/a return -1;
3409n/a
3410n/a if (pid != Py_None) {
3411n/a if (self->bin) {
3412n/a if (save(self, pid, 1) < 0 ||
3413n/a _Pickler_Write(self, &binpersid_op, 1) < 0)
3414n/a goto error;
3415n/a }
3416n/a else {
3417n/a PyObject *pid_str;
3418n/a
3419n/a pid_str = PyObject_Str(pid);
3420n/a if (pid_str == NULL)
3421n/a goto error;
3422n/a
3423n/a /* XXX: Should it check whether the pid contains embedded
3424n/a newlines? */
3425n/a if (!PyUnicode_IS_ASCII(pid_str)) {
3426n/a PyErr_SetString(_Pickle_GetGlobalState()->PicklingError,
3427n/a "persistent IDs in protocol 0 must be "
3428n/a "ASCII strings");
3429n/a Py_DECREF(pid_str);
3430n/a goto error;
3431n/a }
3432n/a
3433n/a if (_Pickler_Write(self, &persid_op, 1) < 0 ||
3434n/a _Pickler_Write(self, PyUnicode_DATA(pid_str),
3435n/a PyUnicode_GET_LENGTH(pid_str)) < 0 ||
3436n/a _Pickler_Write(self, "\n", 1) < 0) {
3437n/a Py_DECREF(pid_str);
3438n/a goto error;
3439n/a }
3440n/a Py_DECREF(pid_str);
3441n/a }
3442n/a status = 1;
3443n/a }
3444n/a
3445n/a if (0) {
3446n/a error:
3447n/a status = -1;
3448n/a }
3449n/a Py_XDECREF(pid);
3450n/a
3451n/a return status;
3452n/a}
3453n/a
3454n/astatic PyObject *
3455n/aget_class(PyObject *obj)
3456n/a{
3457n/a PyObject *cls;
3458n/a _Py_IDENTIFIER(__class__);
3459n/a
3460n/a cls = _PyObject_GetAttrId(obj, &PyId___class__);
3461n/a if (cls == NULL) {
3462n/a if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
3463n/a PyErr_Clear();
3464n/a cls = (PyObject *) Py_TYPE(obj);
3465n/a Py_INCREF(cls);
3466n/a }
3467n/a }
3468n/a return cls;
3469n/a}
3470n/a
3471n/a/* We're saving obj, and args is the 2-thru-5 tuple returned by the
3472n/a * appropriate __reduce__ method for obj.
3473n/a */
3474n/astatic int
3475n/asave_reduce(PicklerObject *self, PyObject *args, PyObject *obj)
3476n/a{
3477n/a PyObject *callable;
3478n/a PyObject *argtup;
3479n/a PyObject *state = NULL;
3480n/a PyObject *listitems = Py_None;
3481n/a PyObject *dictitems = Py_None;
3482n/a PickleState *st = _Pickle_GetGlobalState();
3483n/a Py_ssize_t size;
3484n/a int use_newobj = 0, use_newobj_ex = 0;
3485n/a
3486n/a const char reduce_op = REDUCE;
3487n/a const char build_op = BUILD;
3488n/a const char newobj_op = NEWOBJ;
3489n/a const char newobj_ex_op = NEWOBJ_EX;
3490n/a
3491n/a size = PyTuple_Size(args);
3492n/a if (size < 2 || size > 5) {
3493n/a PyErr_SetString(st->PicklingError, "tuple returned by "
3494n/a "__reduce__ must contain 2 through 5 elements");
3495n/a return -1;
3496n/a }
3497n/a
3498n/a if (!PyArg_UnpackTuple(args, "save_reduce", 2, 5,
3499n/a &callable, &argtup, &state, &listitems, &dictitems))
3500n/a return -1;
3501n/a
3502n/a if (!PyCallable_Check(callable)) {
3503n/a PyErr_SetString(st->PicklingError, "first item of the tuple "
3504n/a "returned by __reduce__ must be callable");
3505n/a return -1;
3506n/a }
3507n/a if (!PyTuple_Check(argtup)) {
3508n/a PyErr_SetString(st->PicklingError, "second item of the tuple "
3509n/a "returned by __reduce__ must be a tuple");
3510n/a return -1;
3511n/a }
3512n/a
3513n/a if (state == Py_None)
3514n/a state = NULL;
3515n/a
3516n/a if (listitems == Py_None)
3517n/a listitems = NULL;
3518n/a else if (!PyIter_Check(listitems)) {
3519n/a PyErr_Format(st->PicklingError, "fourth element of the tuple "
3520n/a "returned by __reduce__ must be an iterator, not %s",
3521n/a Py_TYPE(listitems)->tp_name);
3522n/a return -1;
3523n/a }
3524n/a
3525n/a if (dictitems == Py_None)
3526n/a dictitems = NULL;
3527n/a else if (!PyIter_Check(dictitems)) {
3528n/a PyErr_Format(st->PicklingError, "fifth element of the tuple "
3529n/a "returned by __reduce__ must be an iterator, not %s",
3530n/a Py_TYPE(dictitems)->tp_name);
3531n/a return -1;
3532n/a }
3533n/a
3534n/a if (self->proto >= 2) {
3535n/a PyObject *name;
3536n/a _Py_IDENTIFIER(__name__);
3537n/a
3538n/a name = _PyObject_GetAttrId(callable, &PyId___name__);
3539n/a if (name == NULL) {
3540n/a if (!PyErr_ExceptionMatches(PyExc_AttributeError)) {
3541n/a return -1;
3542n/a }
3543n/a PyErr_Clear();
3544n/a }
3545n/a else if (PyUnicode_Check(name)) {
3546n/a _Py_IDENTIFIER(__newobj_ex__);
3547n/a use_newobj_ex = _PyUnicode_EqualToASCIIId(
3548n/a name, &PyId___newobj_ex__);
3549n/a if (!use_newobj_ex) {
3550n/a _Py_IDENTIFIER(__newobj__);
3551n/a use_newobj = _PyUnicode_EqualToASCIIId(name, &PyId___newobj__);
3552n/a }
3553n/a }
3554n/a Py_XDECREF(name);
3555n/a }
3556n/a
3557n/a if (use_newobj_ex) {
3558n/a PyObject *cls;
3559n/a PyObject *args;
3560n/a PyObject *kwargs;
3561n/a
3562n/a if (Py_SIZE(argtup) != 3) {
3563n/a PyErr_Format(st->PicklingError,
3564n/a "length of the NEWOBJ_EX argument tuple must be "
3565n/a "exactly 3, not %zd", Py_SIZE(argtup));
3566n/a return -1;
3567n/a }
3568n/a
3569n/a cls = PyTuple_GET_ITEM(argtup, 0);
3570n/a if (!PyType_Check(cls)) {
3571n/a PyErr_Format(st->PicklingError,
3572n/a "first item from NEWOBJ_EX argument tuple must "
3573n/a "be a class, not %.200s", Py_TYPE(cls)->tp_name);
3574n/a return -1;
3575n/a }
3576n/a args = PyTuple_GET_ITEM(argtup, 1);
3577n/a if (!PyTuple_Check(args)) {
3578n/a PyErr_Format(st->PicklingError,
3579n/a "second item from NEWOBJ_EX argument tuple must "
3580n/a "be a tuple, not %.200s", Py_TYPE(args)->tp_name);
3581n/a return -1;
3582n/a }
3583n/a kwargs = PyTuple_GET_ITEM(argtup, 2);
3584n/a if (!PyDict_Check(kwargs)) {
3585n/a PyErr_Format(st->PicklingError,
3586n/a "third item from NEWOBJ_EX argument tuple must "
3587n/a "be a dict, not %.200s", Py_TYPE(kwargs)->tp_name);
3588n/a return -1;
3589n/a }
3590n/a
3591n/a if (self->proto >= 4) {
3592n/a if (save(self, cls, 0) < 0 ||
3593n/a save(self, args, 0) < 0 ||
3594n/a save(self, kwargs, 0) < 0 ||
3595n/a _Pickler_Write(self, &newobj_ex_op, 1) < 0) {
3596n/a return -1;
3597n/a }
3598n/a }
3599n/a else {
3600n/a PyObject *newargs;
3601n/a PyObject *cls_new;
3602n/a Py_ssize_t i;
3603n/a _Py_IDENTIFIER(__new__);
3604n/a
3605n/a newargs = PyTuple_New(Py_SIZE(args) + 2);
3606n/a if (newargs == NULL)
3607n/a return -1;
3608n/a
3609n/a cls_new = _PyObject_GetAttrId(cls, &PyId___new__);
3610n/a if (cls_new == NULL) {
3611n/a Py_DECREF(newargs);
3612n/a return -1;
3613n/a }
3614n/a PyTuple_SET_ITEM(newargs, 0, cls_new);
3615n/a Py_INCREF(cls);
3616n/a PyTuple_SET_ITEM(newargs, 1, cls);
3617n/a for (i = 0; i < Py_SIZE(args); i++) {
3618n/a PyObject *item = PyTuple_GET_ITEM(args, i);
3619n/a Py_INCREF(item);
3620n/a PyTuple_SET_ITEM(newargs, i + 2, item);
3621n/a }
3622n/a
3623n/a callable = PyObject_Call(st->partial, newargs, kwargs);
3624n/a Py_DECREF(newargs);
3625n/a if (callable == NULL)
3626n/a return -1;
3627n/a
3628n/a newargs = PyTuple_New(0);
3629n/a if (newargs == NULL) {
3630n/a Py_DECREF(callable);
3631n/a return -1;
3632n/a }
3633n/a
3634n/a if (save(self, callable, 0) < 0 ||
3635n/a save(self, newargs, 0) < 0 ||
3636n/a _Pickler_Write(self, &reduce_op, 1) < 0) {
3637n/a Py_DECREF(newargs);
3638n/a Py_DECREF(callable);
3639n/a return -1;
3640n/a }
3641n/a Py_DECREF(newargs);
3642n/a Py_DECREF(callable);
3643n/a }
3644n/a }
3645n/a else if (use_newobj) {
3646n/a PyObject *cls;
3647n/a PyObject *newargtup;
3648n/a PyObject *obj_class;
3649n/a int p;
3650n/a
3651n/a /* Sanity checks. */
3652n/a if (Py_SIZE(argtup) < 1) {
3653n/a PyErr_SetString(st->PicklingError, "__newobj__ arglist is empty");
3654n/a return -1;
3655n/a }
3656n/a
3657n/a cls = PyTuple_GET_ITEM(argtup, 0);
3658n/a if (!PyType_Check(cls)) {
3659n/a PyErr_SetString(st->PicklingError, "args[0] from "
3660n/a "__newobj__ args is not a type");
3661n/a return -1;
3662n/a }
3663n/a
3664n/a if (obj != NULL) {
3665n/a obj_class = get_class(obj);
3666n/a p = obj_class != cls; /* true iff a problem */
3667n/a Py_DECREF(obj_class);
3668n/a if (p) {
3669n/a PyErr_SetString(st->PicklingError, "args[0] from "
3670n/a "__newobj__ args has the wrong class");
3671n/a return -1;
3672n/a }
3673n/a }
3674n/a /* XXX: These calls save() are prone to infinite recursion. Imagine
3675n/a what happen if the value returned by the __reduce__() method of
3676n/a some extension type contains another object of the same type. Ouch!
3677n/a
3678n/a Here is a quick example, that I ran into, to illustrate what I
3679n/a mean:
3680n/a
3681n/a >>> import pickle, copyreg
3682n/a >>> copyreg.dispatch_table.pop(complex)
3683n/a >>> pickle.dumps(1+2j)
3684n/a Traceback (most recent call last):
3685n/a ...
3686n/a RecursionError: maximum recursion depth exceeded
3687n/a
3688n/a Removing the complex class from copyreg.dispatch_table made the
3689n/a __reduce_ex__() method emit another complex object:
3690n/a
3691n/a >>> (1+1j).__reduce_ex__(2)
3692n/a (<function __newobj__ at 0xb7b71c3c>,
3693n/a (<class 'complex'>, (1+1j)), None, None, None)
3694n/a
3695n/a Thus when save() was called on newargstup (the 2nd item) recursion
3696n/a ensued. Of course, the bug was in the complex class which had a
3697n/a broken __getnewargs__() that emitted another complex object. But,
3698n/a the point, here, is it is quite easy to end up with a broken reduce
3699n/a function. */
3700n/a
3701n/a /* Save the class and its __new__ arguments. */
3702n/a if (save(self, cls, 0) < 0)
3703n/a return -1;
3704n/a
3705n/a newargtup = PyTuple_GetSlice(argtup, 1, Py_SIZE(argtup));
3706n/a if (newargtup == NULL)
3707n/a return -1;
3708n/a
3709n/a p = save(self, newargtup, 0);
3710n/a Py_DECREF(newargtup);
3711n/a if (p < 0)
3712n/a return -1;
3713n/a
3714n/a /* Add NEWOBJ opcode. */
3715n/a if (_Pickler_Write(self, &newobj_op, 1) < 0)
3716n/a return -1;
3717n/a }
3718n/a else { /* Not using NEWOBJ. */
3719n/a if (save(self, callable, 0) < 0 ||
3720n/a save(self, argtup, 0) < 0 ||
3721n/a _Pickler_Write(self, &reduce_op, 1) < 0)
3722n/a return -1;
3723n/a }
3724n/a
3725n/a /* obj can be NULL when save_reduce() is used directly. A NULL obj means
3726n/a the caller do not want to memoize the object. Not particularly useful,
3727n/a but that is to mimic the behavior save_reduce() in pickle.py when
3728n/a obj is None. */
3729n/a if (obj != NULL) {
3730n/a /* If the object is already in the memo, this means it is
3731n/a recursive. In this case, throw away everything we put on the
3732n/a stack, and fetch the object back from the memo. */
3733n/a if (PyMemoTable_Get(self->memo, obj)) {
3734n/a const char pop_op = POP;
3735n/a
3736n/a if (_Pickler_Write(self, &pop_op, 1) < 0)
3737n/a return -1;
3738n/a if (memo_get(self, obj) < 0)
3739n/a return -1;
3740n/a
3741n/a return 0;
3742n/a }
3743n/a else if (memo_put(self, obj) < 0)
3744n/a return -1;
3745n/a }
3746n/a
3747n/a if (listitems && batch_list(self, listitems) < 0)
3748n/a return -1;
3749n/a
3750n/a if (dictitems && batch_dict(self, dictitems) < 0)
3751n/a return -1;
3752n/a
3753n/a if (state) {
3754n/a if (save(self, state, 0) < 0 ||
3755n/a _Pickler_Write(self, &build_op, 1) < 0)
3756n/a return -1;
3757n/a }
3758n/a
3759n/a return 0;
3760n/a}
3761n/a
3762n/astatic int
3763n/asave(PicklerObject *self, PyObject *obj, int pers_save)
3764n/a{
3765n/a PyTypeObject *type;
3766n/a PyObject *reduce_func = NULL;
3767n/a PyObject *reduce_value = NULL;
3768n/a int status = 0;
3769n/a
3770n/a if (_Pickler_OpcodeBoundary(self) < 0)
3771n/a return -1;
3772n/a
3773n/a if (Py_EnterRecursiveCall(" while pickling an object"))
3774n/a return -1;
3775n/a
3776n/a /* The extra pers_save argument is necessary to avoid calling save_pers()
3777n/a on its returned object. */
3778n/a if (!pers_save && self->pers_func) {
3779n/a /* save_pers() returns:
3780n/a -1 to signal an error;
3781n/a 0 if it did nothing successfully;
3782n/a 1 if a persistent id was saved.
3783n/a */
3784n/a if ((status = save_pers(self, obj, self->pers_func)) != 0)
3785n/a goto done;
3786n/a }
3787n/a
3788n/a type = Py_TYPE(obj);
3789n/a
3790n/a /* The old cPickle had an optimization that used switch-case statement
3791n/a dispatching on the first letter of the type name. This has was removed
3792n/a since benchmarks shown that this optimization was actually slowing
3793n/a things down. */
3794n/a
3795n/a /* Atom types; these aren't memoized, so don't check the memo. */
3796n/a
3797n/a if (obj == Py_None) {
3798n/a status = save_none(self, obj);
3799n/a goto done;
3800n/a }
3801n/a else if (obj == Py_False || obj == Py_True) {
3802n/a status = save_bool(self, obj);
3803n/a goto done;
3804n/a }
3805n/a else if (type == &PyLong_Type) {
3806n/a status = save_long(self, obj);
3807n/a goto done;
3808n/a }
3809n/a else if (type == &PyFloat_Type) {
3810n/a status = save_float(self, obj);
3811n/a goto done;
3812n/a }
3813n/a
3814n/a /* Check the memo to see if it has the object. If so, generate
3815n/a a GET (or BINGET) opcode, instead of pickling the object
3816n/a once again. */
3817n/a if (PyMemoTable_Get(self->memo, obj)) {
3818n/a if (memo_get(self, obj) < 0)
3819n/a goto error;
3820n/a goto done;
3821n/a }
3822n/a
3823n/a if (type == &PyBytes_Type) {
3824n/a status = save_bytes(self, obj);
3825n/a goto done;
3826n/a }
3827n/a else if (type == &PyUnicode_Type) {
3828n/a status = save_unicode(self, obj);
3829n/a goto done;
3830n/a }
3831n/a else if (type == &PyDict_Type) {
3832n/a status = save_dict(self, obj);
3833n/a goto done;
3834n/a }
3835n/a else if (type == &PySet_Type) {
3836n/a status = save_set(self, obj);
3837n/a goto done;
3838n/a }
3839n/a else if (type == &PyFrozenSet_Type) {
3840n/a status = save_frozenset(self, obj);
3841n/a goto done;
3842n/a }
3843n/a else if (type == &PyList_Type) {
3844n/a status = save_list(self, obj);
3845n/a goto done;
3846n/a }
3847n/a else if (type == &PyTuple_Type) {
3848n/a status = save_tuple(self, obj);
3849n/a goto done;
3850n/a }
3851n/a else if (type == &PyType_Type) {
3852n/a status = save_type(self, obj);
3853n/a goto done;
3854n/a }
3855n/a else if (type == &PyFunction_Type) {
3856n/a status = save_global(self, obj, NULL);
3857n/a goto done;
3858n/a }
3859n/a
3860n/a /* XXX: This part needs some unit tests. */
3861n/a
3862n/a /* Get a reduction callable, and call it. This may come from
3863n/a * self.dispatch_table, copyreg.dispatch_table, the object's
3864n/a * __reduce_ex__ method, or the object's __reduce__ method.
3865n/a */
3866n/a if (self->dispatch_table == NULL) {
3867n/a PickleState *st = _Pickle_GetGlobalState();
3868n/a reduce_func = PyDict_GetItemWithError(st->dispatch_table,
3869n/a (PyObject *)type);
3870n/a if (reduce_func == NULL) {
3871n/a if (PyErr_Occurred()) {
3872n/a goto error;
3873n/a }
3874n/a } else {
3875n/a /* PyDict_GetItemWithError() returns a borrowed reference.
3876n/a Increase the reference count to be consistent with
3877n/a PyObject_GetItem and _PyObject_GetAttrId used below. */
3878n/a Py_INCREF(reduce_func);
3879n/a }
3880n/a } else {
3881n/a reduce_func = PyObject_GetItem(self->dispatch_table,
3882n/a (PyObject *)type);
3883n/a if (reduce_func == NULL) {
3884n/a if (PyErr_ExceptionMatches(PyExc_KeyError))
3885n/a PyErr_Clear();
3886n/a else
3887n/a goto error;
3888n/a }
3889n/a }
3890n/a if (reduce_func != NULL) {
3891n/a Py_INCREF(obj);
3892n/a reduce_value = _Pickle_FastCall(reduce_func, obj);
3893n/a }
3894n/a else if (PyType_IsSubtype(type, &PyType_Type)) {
3895n/a status = save_global(self, obj, NULL);
3896n/a goto done;
3897n/a }
3898n/a else {
3899n/a _Py_IDENTIFIER(__reduce__);
3900n/a _Py_IDENTIFIER(__reduce_ex__);
3901n/a
3902n/a
3903n/a /* XXX: If the __reduce__ method is defined, __reduce_ex__ is
3904n/a automatically defined as __reduce__. While this is convenient, this
3905n/a make it impossible to know which method was actually called. Of
3906n/a course, this is not a big deal. But still, it would be nice to let
3907n/a the user know which method was called when something go
3908n/a wrong. Incidentally, this means if __reduce_ex__ is not defined, we
3909n/a don't actually have to check for a __reduce__ method. */
3910n/a
3911n/a /* Check for a __reduce_ex__ method. */
3912n/a reduce_func = _PyObject_GetAttrId(obj, &PyId___reduce_ex__);
3913n/a if (reduce_func != NULL) {
3914n/a PyObject *proto;
3915n/a proto = PyLong_FromLong(self->proto);
3916n/a if (proto != NULL) {
3917n/a reduce_value = _Pickle_FastCall(reduce_func, proto);
3918n/a }
3919n/a }
3920n/a else {
3921n/a PickleState *st = _Pickle_GetGlobalState();
3922n/a
3923n/a if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
3924n/a PyErr_Clear();
3925n/a }
3926n/a else {
3927n/a goto error;
3928n/a }
3929n/a /* Check for a __reduce__ method. */
3930n/a reduce_func = _PyObject_GetAttrId(obj, &PyId___reduce__);
3931n/a if (reduce_func != NULL) {
3932n/a reduce_value = _PyObject_CallNoArg(reduce_func);
3933n/a }
3934n/a else {
3935n/a PyErr_Format(st->PicklingError,
3936n/a "can't pickle '%.200s' object: %R",
3937n/a type->tp_name, obj);
3938n/a goto error;
3939n/a }
3940n/a }
3941n/a }
3942n/a
3943n/a if (reduce_value == NULL)
3944n/a goto error;
3945n/a
3946n/a if (PyUnicode_Check(reduce_value)) {
3947n/a status = save_global(self, obj, reduce_value);
3948n/a goto done;
3949n/a }
3950n/a
3951n/a if (!PyTuple_Check(reduce_value)) {
3952n/a PickleState *st = _Pickle_GetGlobalState();
3953n/a PyErr_SetString(st->PicklingError,
3954n/a "__reduce__ must return a string or tuple");
3955n/a goto error;
3956n/a }
3957n/a
3958n/a status = save_reduce(self, reduce_value, obj);
3959n/a
3960n/a if (0) {
3961n/a error:
3962n/a status = -1;
3963n/a }
3964n/a done:
3965n/a
3966n/a Py_LeaveRecursiveCall();
3967n/a Py_XDECREF(reduce_func);
3968n/a Py_XDECREF(reduce_value);
3969n/a
3970n/a return status;
3971n/a}
3972n/a
3973n/astatic int
3974n/adump(PicklerObject *self, PyObject *obj)
3975n/a{
3976n/a const char stop_op = STOP;
3977n/a
3978n/a if (self->proto >= 2) {
3979n/a char header[2];
3980n/a
3981n/a header[0] = PROTO;
3982n/a assert(self->proto >= 0 && self->proto < 256);
3983n/a header[1] = (unsigned char)self->proto;
3984n/a if (_Pickler_Write(self, header, 2) < 0)
3985n/a return -1;
3986n/a if (self->proto >= 4)
3987n/a self->framing = 1;
3988n/a }
3989n/a
3990n/a if (save(self, obj, 0) < 0 ||
3991n/a _Pickler_Write(self, &stop_op, 1) < 0)
3992n/a return -1;
3993n/a
3994n/a return 0;
3995n/a}
3996n/a
3997n/a/*[clinic input]
3998n/a
3999n/a_pickle.Pickler.clear_memo
4000n/a
4001n/aClears the pickler's "memo".
4002n/a
4003n/aThe memo is the data structure that remembers which objects the
4004n/apickler has already seen, so that shared or recursive objects are
4005n/apickled by reference and not by value. This method is useful when
4006n/are-using picklers.
4007n/a[clinic start generated code]*/
4008n/a
4009n/astatic PyObject *
4010n/a_pickle_Pickler_clear_memo_impl(PicklerObject *self)
4011n/a/*[clinic end generated code: output=8665c8658aaa094b input=01bdad52f3d93e56]*/
4012n/a{
4013n/a if (self->memo)
4014n/a PyMemoTable_Clear(self->memo);
4015n/a
4016n/a Py_RETURN_NONE;
4017n/a}
4018n/a
4019n/a/*[clinic input]
4020n/a
4021n/a_pickle.Pickler.dump
4022n/a
4023n/a obj: object
4024n/a /
4025n/a
4026n/aWrite a pickled representation of the given object to the open file.
4027n/a[clinic start generated code]*/
4028n/a
4029n/astatic PyObject *
4030n/a_pickle_Pickler_dump(PicklerObject *self, PyObject *obj)
4031n/a/*[clinic end generated code: output=87ecad1261e02ac7 input=552eb1c0f52260d9]*/
4032n/a{
4033n/a /* Check whether the Pickler was initialized correctly (issue3664).
4034n/a Developers often forget to call __init__() in their subclasses, which
4035n/a would trigger a segfault without this check. */
4036n/a if (self->write == NULL) {
4037n/a PickleState *st = _Pickle_GetGlobalState();
4038n/a PyErr_Format(st->PicklingError,
4039n/a "Pickler.__init__() was not called by %s.__init__()",
4040n/a Py_TYPE(self)->tp_name);
4041n/a return NULL;
4042n/a }
4043n/a
4044n/a if (_Pickler_ClearBuffer(self) < 0)
4045n/a return NULL;
4046n/a
4047n/a if (dump(self, obj) < 0)
4048n/a return NULL;
4049n/a
4050n/a if (_Pickler_FlushToFile(self) < 0)
4051n/a return NULL;
4052n/a
4053n/a Py_RETURN_NONE;
4054n/a}
4055n/a
4056n/a/*[clinic input]
4057n/a
4058n/a_pickle.Pickler.__sizeof__ -> Py_ssize_t
4059n/a
4060n/aReturns size in memory, in bytes.
4061n/a[clinic start generated code]*/
4062n/a
4063n/astatic Py_ssize_t
4064n/a_pickle_Pickler___sizeof___impl(PicklerObject *self)
4065n/a/*[clinic end generated code: output=106edb3123f332e1 input=8cbbec9bd5540d42]*/
4066n/a{
4067n/a Py_ssize_t res, s;
4068n/a
4069n/a res = _PyObject_SIZE(Py_TYPE(self));
4070n/a if (self->memo != NULL) {
4071n/a res += sizeof(PyMemoTable);
4072n/a res += self->memo->mt_allocated * sizeof(PyMemoEntry);
4073n/a }
4074n/a if (self->output_buffer != NULL) {
4075n/a s = _PySys_GetSizeOf(self->output_buffer);
4076n/a if (s == -1)
4077n/a return -1;
4078n/a res += s;
4079n/a }
4080n/a return res;
4081n/a}
4082n/a
4083n/astatic struct PyMethodDef Pickler_methods[] = {
4084n/a _PICKLE_PICKLER_DUMP_METHODDEF
4085n/a _PICKLE_PICKLER_CLEAR_MEMO_METHODDEF
4086n/a _PICKLE_PICKLER___SIZEOF___METHODDEF
4087n/a {NULL, NULL} /* sentinel */
4088n/a};
4089n/a
4090n/astatic void
4091n/aPickler_dealloc(PicklerObject *self)
4092n/a{
4093n/a PyObject_GC_UnTrack(self);
4094n/a
4095n/a Py_XDECREF(self->output_buffer);
4096n/a Py_XDECREF(self->write);
4097n/a Py_XDECREF(self->pers_func);
4098n/a Py_XDECREF(self->dispatch_table);
4099n/a Py_XDECREF(self->fast_memo);
4100n/a
4101n/a PyMemoTable_Del(self->memo);
4102n/a
4103n/a Py_TYPE(self)->tp_free((PyObject *)self);
4104n/a}
4105n/a
4106n/astatic int
4107n/aPickler_traverse(PicklerObject *self, visitproc visit, void *arg)
4108n/a{
4109n/a Py_VISIT(self->write);
4110n/a Py_VISIT(self->pers_func);
4111n/a Py_VISIT(self->dispatch_table);
4112n/a Py_VISIT(self->fast_memo);
4113n/a return 0;
4114n/a}
4115n/a
4116n/astatic int
4117n/aPickler_clear(PicklerObject *self)
4118n/a{
4119n/a Py_CLEAR(self->output_buffer);
4120n/a Py_CLEAR(self->write);
4121n/a Py_CLEAR(self->pers_func);
4122n/a Py_CLEAR(self->dispatch_table);
4123n/a Py_CLEAR(self->fast_memo);
4124n/a
4125n/a if (self->memo != NULL) {
4126n/a PyMemoTable *memo = self->memo;
4127n/a self->memo = NULL;
4128n/a PyMemoTable_Del(memo);
4129n/a }
4130n/a return 0;
4131n/a}
4132n/a
4133n/a
4134n/a/*[clinic input]
4135n/a
4136n/a_pickle.Pickler.__init__
4137n/a
4138n/a file: object
4139n/a protocol: object = NULL
4140n/a fix_imports: bool = True
4141n/a
4142n/aThis takes a binary file for writing a pickle data stream.
4143n/a
4144n/aThe optional *protocol* argument tells the pickler to use the given
4145n/aprotocol; supported protocols are 0, 1, 2, 3 and 4. The default
4146n/aprotocol is 3; a backward-incompatible protocol designed for Python 3.
4147n/a
4148n/aSpecifying a negative protocol version selects the highest protocol
4149n/aversion supported. The higher the protocol used, the more recent the
4150n/aversion of Python needed to read the pickle produced.
4151n/a
4152n/aThe *file* argument must have a write() method that accepts a single
4153n/abytes argument. It can thus be a file object opened for binary
4154n/awriting, an io.BytesIO instance, or any other custom object that meets
4155n/athis interface.
4156n/a
4157n/aIf *fix_imports* is True and protocol is less than 3, pickle will try
4158n/ato map the new Python 3 names to the old module names used in Python
4159n/a2, so that the pickle data stream is readable with Python 2.
4160n/a[clinic start generated code]*/
4161n/a
4162n/astatic int
4163n/a_pickle_Pickler___init___impl(PicklerObject *self, PyObject *file,
4164n/a PyObject *protocol, int fix_imports)
4165n/a/*[clinic end generated code: output=b5f31078dab17fb0 input=4faabdbc763c2389]*/
4166n/a{
4167n/a _Py_IDENTIFIER(persistent_id);
4168n/a _Py_IDENTIFIER(dispatch_table);
4169n/a
4170n/a /* In case of multiple __init__() calls, clear previous content. */
4171n/a if (self->write != NULL)
4172n/a (void)Pickler_clear(self);
4173n/a
4174n/a if (_Pickler_SetProtocol(self, protocol, fix_imports) < 0)
4175n/a return -1;
4176n/a
4177n/a if (_Pickler_SetOutputStream(self, file) < 0)
4178n/a return -1;
4179n/a
4180n/a /* memo and output_buffer may have already been created in _Pickler_New */
4181n/a if (self->memo == NULL) {
4182n/a self->memo = PyMemoTable_New();
4183n/a if (self->memo == NULL)
4184n/a return -1;
4185n/a }
4186n/a self->output_len = 0;
4187n/a if (self->output_buffer == NULL) {
4188n/a self->max_output_len = WRITE_BUF_SIZE;
4189n/a self->output_buffer = PyBytes_FromStringAndSize(NULL,
4190n/a self->max_output_len);
4191n/a if (self->output_buffer == NULL)
4192n/a return -1;
4193n/a }
4194n/a
4195n/a self->fast = 0;
4196n/a self->fast_nesting = 0;
4197n/a self->fast_memo = NULL;
4198n/a self->pers_func = NULL;
4199n/a if (_PyObject_HasAttrId((PyObject *)self, &PyId_persistent_id)) {
4200n/a self->pers_func = _PyObject_GetAttrId((PyObject *)self,
4201n/a &PyId_persistent_id);
4202n/a if (self->pers_func == NULL)
4203n/a return -1;
4204n/a }
4205n/a self->dispatch_table = NULL;
4206n/a if (_PyObject_HasAttrId((PyObject *)self, &PyId_dispatch_table)) {
4207n/a self->dispatch_table = _PyObject_GetAttrId((PyObject *)self,
4208n/a &PyId_dispatch_table);
4209n/a if (self->dispatch_table == NULL)
4210n/a return -1;
4211n/a }
4212n/a
4213n/a return 0;
4214n/a}
4215n/a
4216n/a
4217n/a/* Define a proxy object for the Pickler's internal memo object. This is to
4218n/a * avoid breaking code like:
4219n/a * pickler.memo.clear()
4220n/a * and
4221n/a * pickler.memo = saved_memo
4222n/a * Is this a good idea? Not really, but we don't want to break code that uses
4223n/a * it. Note that we don't implement the entire mapping API here. This is
4224n/a * intentional, as these should be treated as black-box implementation details.
4225n/a */
4226n/a
4227n/a/*[clinic input]
4228n/a_pickle.PicklerMemoProxy.clear
4229n/a
4230n/aRemove all items from memo.
4231n/a[clinic start generated code]*/
4232n/a
4233n/astatic PyObject *
4234n/a_pickle_PicklerMemoProxy_clear_impl(PicklerMemoProxyObject *self)
4235n/a/*[clinic end generated code: output=5fb9370d48ae8b05 input=ccc186dacd0f1405]*/
4236n/a{
4237n/a if (self->pickler->memo)
4238n/a PyMemoTable_Clear(self->pickler->memo);
4239n/a Py_RETURN_NONE;
4240n/a}
4241n/a
4242n/a/*[clinic input]
4243n/a_pickle.PicklerMemoProxy.copy
4244n/a
4245n/aCopy the memo to a new object.
4246n/a[clinic start generated code]*/
4247n/a
4248n/astatic PyObject *
4249n/a_pickle_PicklerMemoProxy_copy_impl(PicklerMemoProxyObject *self)
4250n/a/*[clinic end generated code: output=bb83a919d29225ef input=b73043485ac30b36]*/
4251n/a{
4252n/a Py_ssize_t i;
4253n/a PyMemoTable *memo;
4254n/a PyObject *new_memo = PyDict_New();
4255n/a if (new_memo == NULL)
4256n/a return NULL;
4257n/a
4258n/a memo = self->pickler->memo;
4259n/a for (i = 0; i < memo->mt_allocated; ++i) {
4260n/a PyMemoEntry entry = memo->mt_table[i];
4261n/a if (entry.me_key != NULL) {
4262n/a int status;
4263n/a PyObject *key, *value;
4264n/a
4265n/a key = PyLong_FromVoidPtr(entry.me_key);
4266n/a value = Py_BuildValue("nO", entry.me_value, entry.me_key);
4267n/a
4268n/a if (key == NULL || value == NULL) {
4269n/a Py_XDECREF(key);
4270n/a Py_XDECREF(value);
4271n/a goto error;
4272n/a }
4273n/a status = PyDict_SetItem(new_memo, key, value);
4274n/a Py_DECREF(key);
4275n/a Py_DECREF(value);
4276n/a if (status < 0)
4277n/a goto error;
4278n/a }
4279n/a }
4280n/a return new_memo;
4281n/a
4282n/a error:
4283n/a Py_XDECREF(new_memo);
4284n/a return NULL;
4285n/a}
4286n/a
4287n/a/*[clinic input]
4288n/a_pickle.PicklerMemoProxy.__reduce__
4289n/a
4290n/aImplement pickle support.
4291n/a[clinic start generated code]*/
4292n/a
4293n/astatic PyObject *
4294n/a_pickle_PicklerMemoProxy___reduce___impl(PicklerMemoProxyObject *self)
4295n/a/*[clinic end generated code: output=bebba1168863ab1d input=2f7c540e24b7aae4]*/
4296n/a{
4297n/a PyObject *reduce_value, *dict_args;
4298n/a PyObject *contents = _pickle_PicklerMemoProxy_copy_impl(self);
4299n/a if (contents == NULL)
4300n/a return NULL;
4301n/a
4302n/a reduce_value = PyTuple_New(2);
4303n/a if (reduce_value == NULL) {
4304n/a Py_DECREF(contents);
4305n/a return NULL;
4306n/a }
4307n/a dict_args = PyTuple_New(1);
4308n/a if (dict_args == NULL) {
4309n/a Py_DECREF(contents);
4310n/a Py_DECREF(reduce_value);
4311n/a return NULL;
4312n/a }
4313n/a PyTuple_SET_ITEM(dict_args, 0, contents);
4314n/a Py_INCREF((PyObject *)&PyDict_Type);
4315n/a PyTuple_SET_ITEM(reduce_value, 0, (PyObject *)&PyDict_Type);
4316n/a PyTuple_SET_ITEM(reduce_value, 1, dict_args);
4317n/a return reduce_value;
4318n/a}
4319n/a
4320n/astatic PyMethodDef picklerproxy_methods[] = {
4321n/a _PICKLE_PICKLERMEMOPROXY_CLEAR_METHODDEF
4322n/a _PICKLE_PICKLERMEMOPROXY_COPY_METHODDEF
4323n/a _PICKLE_PICKLERMEMOPROXY___REDUCE___METHODDEF
4324n/a {NULL, NULL} /* sentinel */
4325n/a};
4326n/a
4327n/astatic void
4328n/aPicklerMemoProxy_dealloc(PicklerMemoProxyObject *self)
4329n/a{
4330n/a PyObject_GC_UnTrack(self);
4331n/a Py_XDECREF(self->pickler);
4332n/a PyObject_GC_Del((PyObject *)self);
4333n/a}
4334n/a
4335n/astatic int
4336n/aPicklerMemoProxy_traverse(PicklerMemoProxyObject *self,
4337n/a visitproc visit, void *arg)
4338n/a{
4339n/a Py_VISIT(self->pickler);
4340n/a return 0;
4341n/a}
4342n/a
4343n/astatic int
4344n/aPicklerMemoProxy_clear(PicklerMemoProxyObject *self)
4345n/a{
4346n/a Py_CLEAR(self->pickler);
4347n/a return 0;
4348n/a}
4349n/a
4350n/astatic PyTypeObject PicklerMemoProxyType = {
4351n/a PyVarObject_HEAD_INIT(NULL, 0)
4352n/a "_pickle.PicklerMemoProxy", /*tp_name*/
4353n/a sizeof(PicklerMemoProxyObject), /*tp_basicsize*/
4354n/a 0,
4355n/a (destructor)PicklerMemoProxy_dealloc, /* tp_dealloc */
4356n/a 0, /* tp_print */
4357n/a 0, /* tp_getattr */
4358n/a 0, /* tp_setattr */
4359n/a 0, /* tp_compare */
4360n/a 0, /* tp_repr */
4361n/a 0, /* tp_as_number */
4362n/a 0, /* tp_as_sequence */
4363n/a 0, /* tp_as_mapping */
4364n/a PyObject_HashNotImplemented, /* tp_hash */
4365n/a 0, /* tp_call */
4366n/a 0, /* tp_str */
4367n/a PyObject_GenericGetAttr, /* tp_getattro */
4368n/a PyObject_GenericSetAttr, /* tp_setattro */
4369n/a 0, /* tp_as_buffer */
4370n/a Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
4371n/a 0, /* tp_doc */
4372n/a (traverseproc)PicklerMemoProxy_traverse, /* tp_traverse */
4373n/a (inquiry)PicklerMemoProxy_clear, /* tp_clear */
4374n/a 0, /* tp_richcompare */
4375n/a 0, /* tp_weaklistoffset */
4376n/a 0, /* tp_iter */
4377n/a 0, /* tp_iternext */
4378n/a picklerproxy_methods, /* tp_methods */
4379n/a};
4380n/a
4381n/astatic PyObject *
4382n/aPicklerMemoProxy_New(PicklerObject *pickler)
4383n/a{
4384n/a PicklerMemoProxyObject *self;
4385n/a
4386n/a self = PyObject_GC_New(PicklerMemoProxyObject, &PicklerMemoProxyType);
4387n/a if (self == NULL)
4388n/a return NULL;
4389n/a Py_INCREF(pickler);
4390n/a self->pickler = pickler;
4391n/a PyObject_GC_Track(self);
4392n/a return (PyObject *)self;
4393n/a}
4394n/a
4395n/a/*****************************************************************************/
4396n/a
4397n/astatic PyObject *
4398n/aPickler_get_memo(PicklerObject *self)
4399n/a{
4400n/a return PicklerMemoProxy_New(self);
4401n/a}
4402n/a
4403n/astatic int
4404n/aPickler_set_memo(PicklerObject *self, PyObject *obj)
4405n/a{
4406n/a PyMemoTable *new_memo = NULL;
4407n/a
4408n/a if (obj == NULL) {
4409n/a PyErr_SetString(PyExc_TypeError,
4410n/a "attribute deletion is not supported");
4411n/a return -1;
4412n/a }
4413n/a
4414n/a if (Py_TYPE(obj) == &PicklerMemoProxyType) {
4415n/a PicklerObject *pickler =
4416n/a ((PicklerMemoProxyObject *)obj)->pickler;
4417n/a
4418n/a new_memo = PyMemoTable_Copy(pickler->memo);
4419n/a if (new_memo == NULL)
4420n/a return -1;
4421n/a }
4422n/a else if (PyDict_Check(obj)) {
4423n/a Py_ssize_t i = 0;
4424n/a PyObject *key, *value;
4425n/a
4426n/a new_memo = PyMemoTable_New();
4427n/a if (new_memo == NULL)
4428n/a return -1;
4429n/a
4430n/a while (PyDict_Next(obj, &i, &key, &value)) {
4431n/a Py_ssize_t memo_id;
4432n/a PyObject *memo_obj;
4433n/a
4434n/a if (!PyTuple_Check(value) || Py_SIZE(value) != 2) {
4435n/a PyErr_SetString(PyExc_TypeError,
4436n/a "'memo' values must be 2-item tuples");
4437n/a goto error;
4438n/a }
4439n/a memo_id = PyLong_AsSsize_t(PyTuple_GET_ITEM(value, 0));
4440n/a if (memo_id == -1 && PyErr_Occurred())
4441n/a goto error;
4442n/a memo_obj = PyTuple_GET_ITEM(value, 1);
4443n/a if (PyMemoTable_Set(new_memo, memo_obj, memo_id) < 0)
4444n/a goto error;
4445n/a }
4446n/a }
4447n/a else {
4448n/a PyErr_Format(PyExc_TypeError,
4449n/a "'memo' attribute must be a PicklerMemoProxy object"
4450n/a "or dict, not %.200s", Py_TYPE(obj)->tp_name);
4451n/a return -1;
4452n/a }
4453n/a
4454n/a PyMemoTable_Del(self->memo);
4455n/a self->memo = new_memo;
4456n/a
4457n/a return 0;
4458n/a
4459n/a error:
4460n/a if (new_memo)
4461n/a PyMemoTable_Del(new_memo);
4462n/a return -1;
4463n/a}
4464n/a
4465n/astatic PyObject *
4466n/aPickler_get_persid(PicklerObject *self)
4467n/a{
4468n/a if (self->pers_func == NULL)
4469n/a PyErr_SetString(PyExc_AttributeError, "persistent_id");
4470n/a else
4471n/a Py_INCREF(self->pers_func);
4472n/a return self->pers_func;
4473n/a}
4474n/a
4475n/astatic int
4476n/aPickler_set_persid(PicklerObject *self, PyObject *value)
4477n/a{
4478n/a if (value == NULL) {
4479n/a PyErr_SetString(PyExc_TypeError,
4480n/a "attribute deletion is not supported");
4481n/a return -1;
4482n/a }
4483n/a if (!PyCallable_Check(value)) {
4484n/a PyErr_SetString(PyExc_TypeError,
4485n/a "persistent_id must be a callable taking one argument");
4486n/a return -1;
4487n/a }
4488n/a
4489n/a Py_INCREF(value);
4490n/a Py_XSETREF(self->pers_func, value);
4491n/a
4492n/a return 0;
4493n/a}
4494n/a
4495n/astatic PyMemberDef Pickler_members[] = {
4496n/a {"bin", T_INT, offsetof(PicklerObject, bin)},
4497n/a {"fast", T_INT, offsetof(PicklerObject, fast)},
4498n/a {"dispatch_table", T_OBJECT_EX, offsetof(PicklerObject, dispatch_table)},
4499n/a {NULL}
4500n/a};
4501n/a
4502n/astatic PyGetSetDef Pickler_getsets[] = {
4503n/a {"memo", (getter)Pickler_get_memo,
4504n/a (setter)Pickler_set_memo},
4505n/a {"persistent_id", (getter)Pickler_get_persid,
4506n/a (setter)Pickler_set_persid},
4507n/a {NULL}
4508n/a};
4509n/a
4510n/astatic PyTypeObject Pickler_Type = {
4511n/a PyVarObject_HEAD_INIT(NULL, 0)
4512n/a "_pickle.Pickler" , /*tp_name*/
4513n/a sizeof(PicklerObject), /*tp_basicsize*/
4514n/a 0, /*tp_itemsize*/
4515n/a (destructor)Pickler_dealloc, /*tp_dealloc*/
4516n/a 0, /*tp_print*/
4517n/a 0, /*tp_getattr*/
4518n/a 0, /*tp_setattr*/
4519n/a 0, /*tp_reserved*/
4520n/a 0, /*tp_repr*/
4521n/a 0, /*tp_as_number*/
4522n/a 0, /*tp_as_sequence*/
4523n/a 0, /*tp_as_mapping*/
4524n/a 0, /*tp_hash*/
4525n/a 0, /*tp_call*/
4526n/a 0, /*tp_str*/
4527n/a 0, /*tp_getattro*/
4528n/a 0, /*tp_setattro*/
4529n/a 0, /*tp_as_buffer*/
4530n/a Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
4531n/a _pickle_Pickler___init____doc__, /*tp_doc*/
4532n/a (traverseproc)Pickler_traverse, /*tp_traverse*/
4533n/a (inquiry)Pickler_clear, /*tp_clear*/
4534n/a 0, /*tp_richcompare*/
4535n/a 0, /*tp_weaklistoffset*/
4536n/a 0, /*tp_iter*/
4537n/a 0, /*tp_iternext*/
4538n/a Pickler_methods, /*tp_methods*/
4539n/a Pickler_members, /*tp_members*/
4540n/a Pickler_getsets, /*tp_getset*/
4541n/a 0, /*tp_base*/
4542n/a 0, /*tp_dict*/
4543n/a 0, /*tp_descr_get*/
4544n/a 0, /*tp_descr_set*/
4545n/a 0, /*tp_dictoffset*/
4546n/a _pickle_Pickler___init__, /*tp_init*/
4547n/a PyType_GenericAlloc, /*tp_alloc*/
4548n/a PyType_GenericNew, /*tp_new*/
4549n/a PyObject_GC_Del, /*tp_free*/
4550n/a 0, /*tp_is_gc*/
4551n/a};
4552n/a
4553n/a/* Temporary helper for calling self.find_class().
4554n/a
4555n/a XXX: It would be nice to able to avoid Python function call overhead, by
4556n/a using directly the C version of find_class(), when find_class() is not
4557n/a overridden by a subclass. Although, this could become rather hackish. A
4558n/a simpler optimization would be to call the C function when self is not a
4559n/a subclass instance. */
4560n/astatic PyObject *
4561n/afind_class(UnpicklerObject *self, PyObject *module_name, PyObject *global_name)
4562n/a{
4563n/a _Py_IDENTIFIER(find_class);
4564n/a
4565n/a return _PyObject_CallMethodIdObjArgs((PyObject *)self, &PyId_find_class,
4566n/a module_name, global_name, NULL);
4567n/a}
4568n/a
4569n/astatic Py_ssize_t
4570n/amarker(UnpicklerObject *self)
4571n/a{
4572n/a Py_ssize_t mark;
4573n/a
4574n/a if (self->num_marks < 1) {
4575n/a PickleState *st = _Pickle_GetGlobalState();
4576n/a PyErr_SetString(st->UnpicklingError, "could not find MARK");
4577n/a return -1;
4578n/a }
4579n/a
4580n/a mark = self->marks[--self->num_marks];
4581n/a self->stack->mark_set = self->num_marks != 0;
4582n/a self->stack->fence = self->num_marks ?
4583n/a self->marks[self->num_marks - 1] : 0;
4584n/a return mark;
4585n/a}
4586n/a
4587n/astatic int
4588n/aload_none(UnpicklerObject *self)
4589n/a{
4590n/a PDATA_APPEND(self->stack, Py_None, -1);
4591n/a return 0;
4592n/a}
4593n/a
4594n/astatic int
4595n/aload_int(UnpicklerObject *self)
4596n/a{
4597n/a PyObject *value;
4598n/a char *endptr, *s;
4599n/a Py_ssize_t len;
4600n/a long x;
4601n/a
4602n/a if ((len = _Unpickler_Readline(self, &s)) < 0)
4603n/a return -1;
4604n/a if (len < 2)
4605n/a return bad_readline();
4606n/a
4607n/a errno = 0;
4608n/a /* XXX: Should the base argument of strtol() be explicitly set to 10?
4609n/a XXX(avassalotti): Should this uses PyOS_strtol()? */
4610n/a x = strtol(s, &endptr, 0);
4611n/a
4612n/a if (errno || (*endptr != '\n' && *endptr != '\0')) {
4613n/a /* Hm, maybe we've got something long. Let's try reading
4614n/a * it as a Python int object. */
4615n/a errno = 0;
4616n/a /* XXX: Same thing about the base here. */
4617n/a value = PyLong_FromString(s, NULL, 0);
4618n/a if (value == NULL) {
4619n/a PyErr_SetString(PyExc_ValueError,
4620n/a "could not convert string to int");
4621n/a return -1;
4622n/a }
4623n/a }
4624n/a else {
4625n/a if (len == 3 && (x == 0 || x == 1)) {
4626n/a if ((value = PyBool_FromLong(x)) == NULL)
4627n/a return -1;
4628n/a }
4629n/a else {
4630n/a if ((value = PyLong_FromLong(x)) == NULL)
4631n/a return -1;
4632n/a }
4633n/a }
4634n/a
4635n/a PDATA_PUSH(self->stack, value, -1);
4636n/a return 0;
4637n/a}
4638n/a
4639n/astatic int
4640n/aload_bool(UnpicklerObject *self, PyObject *boolean)
4641n/a{
4642n/a assert(boolean == Py_True || boolean == Py_False);
4643n/a PDATA_APPEND(self->stack, boolean, -1);
4644n/a return 0;
4645n/a}
4646n/a
4647n/a/* s contains x bytes of an unsigned little-endian integer. Return its value
4648n/a * as a C Py_ssize_t, or -1 if it's higher than PY_SSIZE_T_MAX.
4649n/a */
4650n/astatic Py_ssize_t
4651n/acalc_binsize(char *bytes, int nbytes)
4652n/a{
4653n/a unsigned char *s = (unsigned char *)bytes;
4654n/a int i;
4655n/a size_t x = 0;
4656n/a
4657n/a if (nbytes > (int)sizeof(size_t)) {
4658n/a /* Check for integer overflow. BINBYTES8 and BINUNICODE8 opcodes
4659n/a * have 64-bit size that can't be represented on 32-bit platform.
4660n/a */
4661n/a for (i = (int)sizeof(size_t); i < nbytes; i++) {
4662n/a if (s[i])
4663n/a return -1;
4664n/a }
4665n/a nbytes = (int)sizeof(size_t);
4666n/a }
4667n/a for (i = 0; i < nbytes; i++) {
4668n/a x |= (size_t) s[i] << (8 * i);
4669n/a }
4670n/a
4671n/a if (x > PY_SSIZE_T_MAX)
4672n/a return -1;
4673n/a else
4674n/a return (Py_ssize_t) x;
4675n/a}
4676n/a
4677n/a/* s contains x bytes of a little-endian integer. Return its value as a
4678n/a * C int. Obscure: when x is 1 or 2, this is an unsigned little-endian
4679n/a * int, but when x is 4 it's a signed one. This is a historical source
4680n/a * of x-platform bugs.
4681n/a */
4682n/astatic long
4683n/acalc_binint(char *bytes, int nbytes)
4684n/a{
4685n/a unsigned char *s = (unsigned char *)bytes;
4686n/a Py_ssize_t i;
4687n/a long x = 0;
4688n/a
4689n/a for (i = 0; i < nbytes; i++) {
4690n/a x |= (long)s[i] << (8 * i);
4691n/a }
4692n/a
4693n/a /* Unlike BININT1 and BININT2, BININT (more accurately BININT4)
4694n/a * is signed, so on a box with longs bigger than 4 bytes we need
4695n/a * to extend a BININT's sign bit to the full width.
4696n/a */
4697n/a if (SIZEOF_LONG > 4 && nbytes == 4) {
4698n/a x |= -(x & (1L << 31));
4699n/a }
4700n/a
4701n/a return x;
4702n/a}
4703n/a
4704n/astatic int
4705n/aload_binintx(UnpicklerObject *self, char *s, int size)
4706n/a{
4707n/a PyObject *value;
4708n/a long x;
4709n/a
4710n/a x = calc_binint(s, size);
4711n/a
4712n/a if ((value = PyLong_FromLong(x)) == NULL)
4713n/a return -1;
4714n/a
4715n/a PDATA_PUSH(self->stack, value, -1);
4716n/a return 0;
4717n/a}
4718n/a
4719n/astatic int
4720n/aload_binint(UnpicklerObject *self)
4721n/a{
4722n/a char *s;
4723n/a
4724n/a if (_Unpickler_Read(self, &s, 4) < 0)
4725n/a return -1;
4726n/a
4727n/a return load_binintx(self, s, 4);
4728n/a}
4729n/a
4730n/astatic int
4731n/aload_binint1(UnpicklerObject *self)
4732n/a{
4733n/a char *s;
4734n/a
4735n/a if (_Unpickler_Read(self, &s, 1) < 0)
4736n/a return -1;
4737n/a
4738n/a return load_binintx(self, s, 1);
4739n/a}
4740n/a
4741n/astatic int
4742n/aload_binint2(UnpicklerObject *self)
4743n/a{
4744n/a char *s;
4745n/a
4746n/a if (_Unpickler_Read(self, &s, 2) < 0)
4747n/a return -1;
4748n/a
4749n/a return load_binintx(self, s, 2);
4750n/a}
4751n/a
4752n/astatic int
4753n/aload_long(UnpicklerObject *self)
4754n/a{
4755n/a PyObject *value;
4756n/a char *s = NULL;
4757n/a Py_ssize_t len;
4758n/a
4759n/a if ((len = _Unpickler_Readline(self, &s)) < 0)
4760n/a return -1;
4761n/a if (len < 2)
4762n/a return bad_readline();
4763n/a
4764n/a /* s[len-2] will usually be 'L' (and s[len-1] is '\n'); we need to remove
4765n/a the 'L' before calling PyLong_FromString. In order to maintain
4766n/a compatibility with Python 3.0.0, we don't actually *require*
4767n/a the 'L' to be present. */
4768n/a if (s[len-2] == 'L')
4769n/a s[len-2] = '\0';
4770n/a /* XXX: Should the base argument explicitly set to 10? */
4771n/a value = PyLong_FromString(s, NULL, 0);
4772n/a if (value == NULL)
4773n/a return -1;
4774n/a
4775n/a PDATA_PUSH(self->stack, value, -1);
4776n/a return 0;
4777n/a}
4778n/a
4779n/a/* 'size' bytes contain the # of bytes of little-endian 256's-complement
4780n/a * data following.
4781n/a */
4782n/astatic int
4783n/aload_counted_long(UnpicklerObject *self, int size)
4784n/a{
4785n/a PyObject *value;
4786n/a char *nbytes;
4787n/a char *pdata;
4788n/a
4789n/a assert(size == 1 || size == 4);
4790n/a if (_Unpickler_Read(self, &nbytes, size) < 0)
4791n/a return -1;
4792n/a
4793n/a size = calc_binint(nbytes, size);
4794n/a if (size < 0) {
4795n/a PickleState *st = _Pickle_GetGlobalState();
4796n/a /* Corrupt or hostile pickle -- we never write one like this */
4797n/a PyErr_SetString(st->UnpicklingError,
4798n/a "LONG pickle has negative byte count");
4799n/a return -1;
4800n/a }
4801n/a
4802n/a if (size == 0)
4803n/a value = PyLong_FromLong(0L);
4804n/a else {
4805n/a /* Read the raw little-endian bytes and convert. */
4806n/a if (_Unpickler_Read(self, &pdata, size) < 0)
4807n/a return -1;
4808n/a value = _PyLong_FromByteArray((unsigned char *)pdata, (size_t)size,
4809n/a 1 /* little endian */ , 1 /* signed */ );
4810n/a }
4811n/a if (value == NULL)
4812n/a return -1;
4813n/a PDATA_PUSH(self->stack, value, -1);
4814n/a return 0;
4815n/a}
4816n/a
4817n/astatic int
4818n/aload_float(UnpicklerObject *self)
4819n/a{
4820n/a PyObject *value;
4821n/a char *endptr, *s;
4822n/a Py_ssize_t len;
4823n/a double d;
4824n/a
4825n/a if ((len = _Unpickler_Readline(self, &s)) < 0)
4826n/a return -1;
4827n/a if (len < 2)
4828n/a return bad_readline();
4829n/a
4830n/a errno = 0;
4831n/a d = PyOS_string_to_double(s, &endptr, PyExc_OverflowError);
4832n/a if (d == -1.0 && PyErr_Occurred())
4833n/a return -1;
4834n/a if ((endptr[0] != '\n') && (endptr[0] != '\0')) {
4835n/a PyErr_SetString(PyExc_ValueError, "could not convert string to float");
4836n/a return -1;
4837n/a }
4838n/a value = PyFloat_FromDouble(d);
4839n/a if (value == NULL)
4840n/a return -1;
4841n/a
4842n/a PDATA_PUSH(self->stack, value, -1);
4843n/a return 0;
4844n/a}
4845n/a
4846n/astatic int
4847n/aload_binfloat(UnpicklerObject *self)
4848n/a{
4849n/a PyObject *value;
4850n/a double x;
4851n/a char *s;
4852n/a
4853n/a if (_Unpickler_Read(self, &s, 8) < 0)
4854n/a return -1;
4855n/a
4856n/a x = _PyFloat_Unpack8((unsigned char *)s, 0);
4857n/a if (x == -1.0 && PyErr_Occurred())
4858n/a return -1;
4859n/a
4860n/a if ((value = PyFloat_FromDouble(x)) == NULL)
4861n/a return -1;
4862n/a
4863n/a PDATA_PUSH(self->stack, value, -1);
4864n/a return 0;
4865n/a}
4866n/a
4867n/astatic int
4868n/aload_string(UnpicklerObject *self)
4869n/a{
4870n/a PyObject *bytes;
4871n/a PyObject *obj;
4872n/a Py_ssize_t len;
4873n/a char *s, *p;
4874n/a
4875n/a if ((len = _Unpickler_Readline(self, &s)) < 0)
4876n/a return -1;
4877n/a /* Strip the newline */
4878n/a len--;
4879n/a /* Strip outermost quotes */
4880n/a if (len >= 2 && s[0] == s[len - 1] && (s[0] == '\'' || s[0] == '"')) {
4881n/a p = s + 1;
4882n/a len -= 2;
4883n/a }
4884n/a else {
4885n/a PickleState *st = _Pickle_GetGlobalState();
4886n/a PyErr_SetString(st->UnpicklingError,
4887n/a "the STRING opcode argument must be quoted");
4888n/a return -1;
4889n/a }
4890n/a assert(len >= 0);
4891n/a
4892n/a /* Use the PyBytes API to decode the string, since that is what is used
4893n/a to encode, and then coerce the result to Unicode. */
4894n/a bytes = PyBytes_DecodeEscape(p, len, NULL, 0, NULL);
4895n/a if (bytes == NULL)
4896n/a return -1;
4897n/a
4898n/a /* Leave the Python 2.x strings as bytes if the *encoding* given to the
4899n/a Unpickler was 'bytes'. Otherwise, convert them to unicode. */
4900n/a if (strcmp(self->encoding, "bytes") == 0) {
4901n/a obj = bytes;
4902n/a }
4903n/a else {
4904n/a obj = PyUnicode_FromEncodedObject(bytes, self->encoding, self->errors);
4905n/a Py_DECREF(bytes);
4906n/a if (obj == NULL) {
4907n/a return -1;
4908n/a }
4909n/a }
4910n/a
4911n/a PDATA_PUSH(self->stack, obj, -1);
4912n/a return 0;
4913n/a}
4914n/a
4915n/astatic int
4916n/aload_counted_binstring(UnpicklerObject *self, int nbytes)
4917n/a{
4918n/a PyObject *obj;
4919n/a Py_ssize_t size;
4920n/a char *s;
4921n/a
4922n/a if (_Unpickler_Read(self, &s, nbytes) < 0)
4923n/a return -1;
4924n/a
4925n/a size = calc_binsize(s, nbytes);
4926n/a if (size < 0) {
4927n/a PickleState *st = _Pickle_GetGlobalState();
4928n/a PyErr_Format(st->UnpicklingError,
4929n/a "BINSTRING exceeds system's maximum size of %zd bytes",
4930n/a PY_SSIZE_T_MAX);
4931n/a return -1;
4932n/a }
4933n/a
4934n/a if (_Unpickler_Read(self, &s, size) < 0)
4935n/a return -1;
4936n/a
4937n/a /* Convert Python 2.x strings to bytes if the *encoding* given to the
4938n/a Unpickler was 'bytes'. Otherwise, convert them to unicode. */
4939n/a if (strcmp(self->encoding, "bytes") == 0) {
4940n/a obj = PyBytes_FromStringAndSize(s, size);
4941n/a }
4942n/a else {
4943n/a obj = PyUnicode_Decode(s, size, self->encoding, self->errors);
4944n/a }
4945n/a if (obj == NULL) {
4946n/a return -1;
4947n/a }
4948n/a
4949n/a PDATA_PUSH(self->stack, obj, -1);
4950n/a return 0;
4951n/a}
4952n/a
4953n/astatic int
4954n/aload_counted_binbytes(UnpicklerObject *self, int nbytes)
4955n/a{
4956n/a PyObject *bytes;
4957n/a Py_ssize_t size;
4958n/a char *s;
4959n/a
4960n/a if (_Unpickler_Read(self, &s, nbytes) < 0)
4961n/a return -1;
4962n/a
4963n/a size = calc_binsize(s, nbytes);
4964n/a if (size < 0) {
4965n/a PyErr_Format(PyExc_OverflowError,
4966n/a "BINBYTES exceeds system's maximum size of %zd bytes",
4967n/a PY_SSIZE_T_MAX);
4968n/a return -1;
4969n/a }
4970n/a
4971n/a if (_Unpickler_Read(self, &s, size) < 0)
4972n/a return -1;
4973n/a
4974n/a bytes = PyBytes_FromStringAndSize(s, size);
4975n/a if (bytes == NULL)
4976n/a return -1;
4977n/a
4978n/a PDATA_PUSH(self->stack, bytes, -1);
4979n/a return 0;
4980n/a}
4981n/a
4982n/astatic int
4983n/aload_unicode(UnpicklerObject *self)
4984n/a{
4985n/a PyObject *str;
4986n/a Py_ssize_t len;
4987n/a char *s = NULL;
4988n/a
4989n/a if ((len = _Unpickler_Readline(self, &s)) < 0)
4990n/a return -1;
4991n/a if (len < 1)
4992n/a return bad_readline();
4993n/a
4994n/a str = PyUnicode_DecodeRawUnicodeEscape(s, len - 1, NULL);
4995n/a if (str == NULL)
4996n/a return -1;
4997n/a
4998n/a PDATA_PUSH(self->stack, str, -1);
4999n/a return 0;
5000n/a}
5001n/a
5002n/astatic int
5003n/aload_counted_binunicode(UnpicklerObject *self, int nbytes)
5004n/a{
5005n/a PyObject *str;
5006n/a Py_ssize_t size;
5007n/a char *s;
5008n/a
5009n/a if (_Unpickler_Read(self, &s, nbytes) < 0)
5010n/a return -1;
5011n/a
5012n/a size = calc_binsize(s, nbytes);
5013n/a if (size < 0) {
5014n/a PyErr_Format(PyExc_OverflowError,
5015n/a "BINUNICODE exceeds system's maximum size of %zd bytes",
5016n/a PY_SSIZE_T_MAX);
5017n/a return -1;
5018n/a }
5019n/a
5020n/a if (_Unpickler_Read(self, &s, size) < 0)
5021n/a return -1;
5022n/a
5023n/a str = PyUnicode_DecodeUTF8(s, size, "surrogatepass");
5024n/a if (str == NULL)
5025n/a return -1;
5026n/a
5027n/a PDATA_PUSH(self->stack, str, -1);
5028n/a return 0;
5029n/a}
5030n/a
5031n/astatic int
5032n/aload_counted_tuple(UnpicklerObject *self, Py_ssize_t len)
5033n/a{
5034n/a PyObject *tuple;
5035n/a
5036n/a if (Py_SIZE(self->stack) < len)
5037n/a return Pdata_stack_underflow(self->stack);
5038n/a
5039n/a tuple = Pdata_poptuple(self->stack, Py_SIZE(self->stack) - len);
5040n/a if (tuple == NULL)
5041n/a return -1;
5042n/a PDATA_PUSH(self->stack, tuple, -1);
5043n/a return 0;
5044n/a}
5045n/a
5046n/astatic int
5047n/aload_tuple(UnpicklerObject *self)
5048n/a{
5049n/a Py_ssize_t i;
5050n/a
5051n/a if ((i = marker(self)) < 0)
5052n/a return -1;
5053n/a
5054n/a return load_counted_tuple(self, Py_SIZE(self->stack) - i);
5055n/a}
5056n/a
5057n/astatic int
5058n/aload_empty_list(UnpicklerObject *self)
5059n/a{
5060n/a PyObject *list;
5061n/a
5062n/a if ((list = PyList_New(0)) == NULL)
5063n/a return -1;
5064n/a PDATA_PUSH(self->stack, list, -1);
5065n/a return 0;
5066n/a}
5067n/a
5068n/astatic int
5069n/aload_empty_dict(UnpicklerObject *self)
5070n/a{
5071n/a PyObject *dict;
5072n/a
5073n/a if ((dict = PyDict_New()) == NULL)
5074n/a return -1;
5075n/a PDATA_PUSH(self->stack, dict, -1);
5076n/a return 0;
5077n/a}
5078n/a
5079n/astatic int
5080n/aload_empty_set(UnpicklerObject *self)
5081n/a{
5082n/a PyObject *set;
5083n/a
5084n/a if ((set = PySet_New(NULL)) == NULL)
5085n/a return -1;
5086n/a PDATA_PUSH(self->stack, set, -1);
5087n/a return 0;
5088n/a}
5089n/a
5090n/astatic int
5091n/aload_list(UnpicklerObject *self)
5092n/a{
5093n/a PyObject *list;
5094n/a Py_ssize_t i;
5095n/a
5096n/a if ((i = marker(self)) < 0)
5097n/a return -1;
5098n/a
5099n/a list = Pdata_poplist(self->stack, i);
5100n/a if (list == NULL)
5101n/a return -1;
5102n/a PDATA_PUSH(self->stack, list, -1);
5103n/a return 0;
5104n/a}
5105n/a
5106n/astatic int
5107n/aload_dict(UnpicklerObject *self)
5108n/a{
5109n/a PyObject *dict, *key, *value;
5110n/a Py_ssize_t i, j, k;
5111n/a
5112n/a if ((i = marker(self)) < 0)
5113n/a return -1;
5114n/a j = Py_SIZE(self->stack);
5115n/a
5116n/a if ((dict = PyDict_New()) == NULL)
5117n/a return -1;
5118n/a
5119n/a if ((j - i) % 2 != 0) {
5120n/a PickleState *st = _Pickle_GetGlobalState();
5121n/a PyErr_SetString(st->UnpicklingError, "odd number of items for DICT");
5122n/a Py_DECREF(dict);
5123n/a return -1;
5124n/a }
5125n/a
5126n/a for (k = i + 1; k < j; k += 2) {
5127n/a key = self->stack->data[k - 1];
5128n/a value = self->stack->data[k];
5129n/a if (PyDict_SetItem(dict, key, value) < 0) {
5130n/a Py_DECREF(dict);
5131n/a return -1;
5132n/a }
5133n/a }
5134n/a Pdata_clear(self->stack, i);
5135n/a PDATA_PUSH(self->stack, dict, -1);
5136n/a return 0;
5137n/a}
5138n/a
5139n/astatic int
5140n/aload_frozenset(UnpicklerObject *self)
5141n/a{
5142n/a PyObject *items;
5143n/a PyObject *frozenset;
5144n/a Py_ssize_t i;
5145n/a
5146n/a if ((i = marker(self)) < 0)
5147n/a return -1;
5148n/a
5149n/a items = Pdata_poptuple(self->stack, i);
5150n/a if (items == NULL)
5151n/a return -1;
5152n/a
5153n/a frozenset = PyFrozenSet_New(items);
5154n/a Py_DECREF(items);
5155n/a if (frozenset == NULL)
5156n/a return -1;
5157n/a
5158n/a PDATA_PUSH(self->stack, frozenset, -1);
5159n/a return 0;
5160n/a}
5161n/a
5162n/astatic PyObject *
5163n/ainstantiate(PyObject *cls, PyObject *args)
5164n/a{
5165n/a PyObject *result = NULL;
5166n/a _Py_IDENTIFIER(__getinitargs__);
5167n/a /* Caller must assure args are a tuple. Normally, args come from
5168n/a Pdata_poptuple which packs objects from the top of the stack
5169n/a into a newly created tuple. */
5170n/a assert(PyTuple_Check(args));
5171n/a if (Py_SIZE(args) > 0 || !PyType_Check(cls) ||
5172n/a _PyObject_HasAttrId(cls, &PyId___getinitargs__)) {
5173n/a result = PyObject_CallObject(cls, args);
5174n/a }
5175n/a else {
5176n/a _Py_IDENTIFIER(__new__);
5177n/a
5178n/a result = _PyObject_CallMethodIdObjArgs(cls, &PyId___new__, cls, NULL);
5179n/a }
5180n/a return result;
5181n/a}
5182n/a
5183n/astatic int
5184n/aload_obj(UnpicklerObject *self)
5185n/a{
5186n/a PyObject *cls, *args, *obj = NULL;
5187n/a Py_ssize_t i;
5188n/a
5189n/a if ((i = marker(self)) < 0)
5190n/a return -1;
5191n/a
5192n/a if (Py_SIZE(self->stack) - i < 1)
5193n/a return Pdata_stack_underflow(self->stack);
5194n/a
5195n/a args = Pdata_poptuple(self->stack, i + 1);
5196n/a if (args == NULL)
5197n/a return -1;
5198n/a
5199n/a PDATA_POP(self->stack, cls);
5200n/a if (cls) {
5201n/a obj = instantiate(cls, args);
5202n/a Py_DECREF(cls);
5203n/a }
5204n/a Py_DECREF(args);
5205n/a if (obj == NULL)
5206n/a return -1;
5207n/a
5208n/a PDATA_PUSH(self->stack, obj, -1);
5209n/a return 0;
5210n/a}
5211n/a
5212n/astatic int
5213n/aload_inst(UnpicklerObject *self)
5214n/a{
5215n/a PyObject *cls = NULL;
5216n/a PyObject *args = NULL;
5217n/a PyObject *obj = NULL;
5218n/a PyObject *module_name;
5219n/a PyObject *class_name;
5220n/a Py_ssize_t len;
5221n/a Py_ssize_t i;
5222n/a char *s;
5223n/a
5224n/a if ((i = marker(self)) < 0)
5225n/a return -1;
5226n/a if ((len = _Unpickler_Readline(self, &s)) < 0)
5227n/a return -1;
5228n/a if (len < 2)
5229n/a return bad_readline();
5230n/a
5231n/a /* Here it is safe to use PyUnicode_DecodeASCII(), even though non-ASCII
5232n/a identifiers are permitted in Python 3.0, since the INST opcode is only
5233n/a supported by older protocols on Python 2.x. */
5234n/a module_name = PyUnicode_DecodeASCII(s, len - 1, "strict");
5235n/a if (module_name == NULL)
5236n/a return -1;
5237n/a
5238n/a if ((len = _Unpickler_Readline(self, &s)) >= 0) {
5239n/a if (len < 2) {
5240n/a Py_DECREF(module_name);
5241n/a return bad_readline();
5242n/a }
5243n/a class_name = PyUnicode_DecodeASCII(s, len - 1, "strict");
5244n/a if (class_name != NULL) {
5245n/a cls = find_class(self, module_name, class_name);
5246n/a Py_DECREF(class_name);
5247n/a }
5248n/a }
5249n/a Py_DECREF(module_name);
5250n/a
5251n/a if (cls == NULL)
5252n/a return -1;
5253n/a
5254n/a if ((args = Pdata_poptuple(self->stack, i)) != NULL) {
5255n/a obj = instantiate(cls, args);
5256n/a Py_DECREF(args);
5257n/a }
5258n/a Py_DECREF(cls);
5259n/a
5260n/a if (obj == NULL)
5261n/a return -1;
5262n/a
5263n/a PDATA_PUSH(self->stack, obj, -1);
5264n/a return 0;
5265n/a}
5266n/a
5267n/astatic int
5268n/aload_newobj(UnpicklerObject *self)
5269n/a{
5270n/a PyObject *args = NULL;
5271n/a PyObject *clsraw = NULL;
5272n/a PyTypeObject *cls; /* clsraw cast to its true type */
5273n/a PyObject *obj;
5274n/a PickleState *st = _Pickle_GetGlobalState();
5275n/a
5276n/a /* Stack is ... cls argtuple, and we want to call
5277n/a * cls.__new__(cls, *argtuple).
5278n/a */
5279n/a PDATA_POP(self->stack, args);
5280n/a if (args == NULL)
5281n/a goto error;
5282n/a if (!PyTuple_Check(args)) {
5283n/a PyErr_SetString(st->UnpicklingError,
5284n/a "NEWOBJ expected an arg " "tuple.");
5285n/a goto error;
5286n/a }
5287n/a
5288n/a PDATA_POP(self->stack, clsraw);
5289n/a cls = (PyTypeObject *)clsraw;
5290n/a if (cls == NULL)
5291n/a goto error;
5292n/a if (!PyType_Check(cls)) {
5293n/a PyErr_SetString(st->UnpicklingError, "NEWOBJ class argument "
5294n/a "isn't a type object");
5295n/a goto error;
5296n/a }
5297n/a if (cls->tp_new == NULL) {
5298n/a PyErr_SetString(st->UnpicklingError, "NEWOBJ class argument "
5299n/a "has NULL tp_new");
5300n/a goto error;
5301n/a }
5302n/a
5303n/a /* Call __new__. */
5304n/a obj = cls->tp_new(cls, args, NULL);
5305n/a if (obj == NULL)
5306n/a goto error;
5307n/a
5308n/a Py_DECREF(args);
5309n/a Py_DECREF(clsraw);
5310n/a PDATA_PUSH(self->stack, obj, -1);
5311n/a return 0;
5312n/a
5313n/a error:
5314n/a Py_XDECREF(args);
5315n/a Py_XDECREF(clsraw);
5316n/a return -1;
5317n/a}
5318n/a
5319n/astatic int
5320n/aload_newobj_ex(UnpicklerObject *self)
5321n/a{
5322n/a PyObject *cls, *args, *kwargs;
5323n/a PyObject *obj;
5324n/a PickleState *st = _Pickle_GetGlobalState();
5325n/a
5326n/a PDATA_POP(self->stack, kwargs);
5327n/a if (kwargs == NULL) {
5328n/a return -1;
5329n/a }
5330n/a PDATA_POP(self->stack, args);
5331n/a if (args == NULL) {
5332n/a Py_DECREF(kwargs);
5333n/a return -1;
5334n/a }
5335n/a PDATA_POP(self->stack, cls);
5336n/a if (cls == NULL) {
5337n/a Py_DECREF(kwargs);
5338n/a Py_DECREF(args);
5339n/a return -1;
5340n/a }
5341n/a
5342n/a if (!PyType_Check(cls)) {
5343n/a Py_DECREF(kwargs);
5344n/a Py_DECREF(args);
5345n/a PyErr_Format(st->UnpicklingError,
5346n/a "NEWOBJ_EX class argument must be a type, not %.200s",
5347n/a Py_TYPE(cls)->tp_name);
5348n/a Py_DECREF(cls);
5349n/a return -1;
5350n/a }
5351n/a
5352n/a if (((PyTypeObject *)cls)->tp_new == NULL) {
5353n/a Py_DECREF(kwargs);
5354n/a Py_DECREF(args);
5355n/a Py_DECREF(cls);
5356n/a PyErr_SetString(st->UnpicklingError,
5357n/a "NEWOBJ_EX class argument doesn't have __new__");
5358n/a return -1;
5359n/a }
5360n/a obj = ((PyTypeObject *)cls)->tp_new((PyTypeObject *)cls, args, kwargs);
5361n/a Py_DECREF(kwargs);
5362n/a Py_DECREF(args);
5363n/a Py_DECREF(cls);
5364n/a if (obj == NULL) {
5365n/a return -1;
5366n/a }
5367n/a PDATA_PUSH(self->stack, obj, -1);
5368n/a return 0;
5369n/a}
5370n/a
5371n/astatic int
5372n/aload_global(UnpicklerObject *self)
5373n/a{
5374n/a PyObject *global = NULL;
5375n/a PyObject *module_name;
5376n/a PyObject *global_name;
5377n/a Py_ssize_t len;
5378n/a char *s;
5379n/a
5380n/a if ((len = _Unpickler_Readline(self, &s)) < 0)
5381n/a return -1;
5382n/a if (len < 2)
5383n/a return bad_readline();
5384n/a module_name = PyUnicode_DecodeUTF8(s, len - 1, "strict");
5385n/a if (!module_name)
5386n/a return -1;
5387n/a
5388n/a if ((len = _Unpickler_Readline(self, &s)) >= 0) {
5389n/a if (len < 2) {
5390n/a Py_DECREF(module_name);
5391n/a return bad_readline();
5392n/a }
5393n/a global_name = PyUnicode_DecodeUTF8(s, len - 1, "strict");
5394n/a if (global_name) {
5395n/a global = find_class(self, module_name, global_name);
5396n/a Py_DECREF(global_name);
5397n/a }
5398n/a }
5399n/a Py_DECREF(module_name);
5400n/a
5401n/a if (global == NULL)
5402n/a return -1;
5403n/a PDATA_PUSH(self->stack, global, -1);
5404n/a return 0;
5405n/a}
5406n/a
5407n/astatic int
5408n/aload_stack_global(UnpicklerObject *self)
5409n/a{
5410n/a PyObject *global;
5411n/a PyObject *module_name;
5412n/a PyObject *global_name;
5413n/a
5414n/a PDATA_POP(self->stack, global_name);
5415n/a PDATA_POP(self->stack, module_name);
5416n/a if (module_name == NULL || !PyUnicode_CheckExact(module_name) ||
5417n/a global_name == NULL || !PyUnicode_CheckExact(global_name)) {
5418n/a PickleState *st = _Pickle_GetGlobalState();
5419n/a PyErr_SetString(st->UnpicklingError, "STACK_GLOBAL requires str");
5420n/a Py_XDECREF(global_name);
5421n/a Py_XDECREF(module_name);
5422n/a return -1;
5423n/a }
5424n/a global = find_class(self, module_name, global_name);
5425n/a Py_DECREF(global_name);
5426n/a Py_DECREF(module_name);
5427n/a if (global == NULL)
5428n/a return -1;
5429n/a PDATA_PUSH(self->stack, global, -1);
5430n/a return 0;
5431n/a}
5432n/a
5433n/astatic int
5434n/aload_persid(UnpicklerObject *self)
5435n/a{
5436n/a PyObject *pid;
5437n/a Py_ssize_t len;
5438n/a char *s;
5439n/a
5440n/a if (self->pers_func) {
5441n/a if ((len = _Unpickler_Readline(self, &s)) < 0)
5442n/a return -1;
5443n/a if (len < 1)
5444n/a return bad_readline();
5445n/a
5446n/a pid = PyUnicode_DecodeASCII(s, len - 1, "strict");
5447n/a if (pid == NULL) {
5448n/a if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
5449n/a PyErr_SetString(_Pickle_GetGlobalState()->UnpicklingError,
5450n/a "persistent IDs in protocol 0 must be "
5451n/a "ASCII strings");
5452n/a }
5453n/a return -1;
5454n/a }
5455n/a
5456n/a /* This does not leak since _Pickle_FastCall() steals the reference
5457n/a to pid first. */
5458n/a pid = _Pickle_FastCall(self->pers_func, pid);
5459n/a if (pid == NULL)
5460n/a return -1;
5461n/a
5462n/a PDATA_PUSH(self->stack, pid, -1);
5463n/a return 0;
5464n/a }
5465n/a else {
5466n/a PickleState *st = _Pickle_GetGlobalState();
5467n/a PyErr_SetString(st->UnpicklingError,
5468n/a "A load persistent id instruction was encountered,\n"
5469n/a "but no persistent_load function was specified.");
5470n/a return -1;
5471n/a }
5472n/a}
5473n/a
5474n/astatic int
5475n/aload_binpersid(UnpicklerObject *self)
5476n/a{
5477n/a PyObject *pid;
5478n/a
5479n/a if (self->pers_func) {
5480n/a PDATA_POP(self->stack, pid);
5481n/a if (pid == NULL)
5482n/a return -1;
5483n/a
5484n/a /* This does not leak since _Pickle_FastCall() steals the
5485n/a reference to pid first. */
5486n/a pid = _Pickle_FastCall(self->pers_func, pid);
5487n/a if (pid == NULL)
5488n/a return -1;
5489n/a
5490n/a PDATA_PUSH(self->stack, pid, -1);
5491n/a return 0;
5492n/a }
5493n/a else {
5494n/a PickleState *st = _Pickle_GetGlobalState();
5495n/a PyErr_SetString(st->UnpicklingError,
5496n/a "A load persistent id instruction was encountered,\n"
5497n/a "but no persistent_load function was specified.");
5498n/a return -1;
5499n/a }
5500n/a}
5501n/a
5502n/astatic int
5503n/aload_pop(UnpicklerObject *self)
5504n/a{
5505n/a Py_ssize_t len = Py_SIZE(self->stack);
5506n/a
5507n/a /* Note that we split the (pickle.py) stack into two stacks,
5508n/a * an object stack and a mark stack. We have to be clever and
5509n/a * pop the right one. We do this by looking at the top of the
5510n/a * mark stack first, and only signalling a stack underflow if
5511n/a * the object stack is empty and the mark stack doesn't match
5512n/a * our expectations.
5513n/a */
5514n/a if (self->num_marks > 0 && self->marks[self->num_marks - 1] == len) {
5515n/a self->num_marks--;
5516n/a self->stack->mark_set = self->num_marks != 0;
5517n/a self->stack->fence = self->num_marks ?
5518n/a self->marks[self->num_marks - 1] : 0;
5519n/a } else if (len <= self->stack->fence)
5520n/a return Pdata_stack_underflow(self->stack);
5521n/a else {
5522n/a len--;
5523n/a Py_DECREF(self->stack->data[len]);
5524n/a Py_SIZE(self->stack) = len;
5525n/a }
5526n/a return 0;
5527n/a}
5528n/a
5529n/astatic int
5530n/aload_pop_mark(UnpicklerObject *self)
5531n/a{
5532n/a Py_ssize_t i;
5533n/a
5534n/a if ((i = marker(self)) < 0)
5535n/a return -1;
5536n/a
5537n/a Pdata_clear(self->stack, i);
5538n/a
5539n/a return 0;
5540n/a}
5541n/a
5542n/astatic int
5543n/aload_dup(UnpicklerObject *self)
5544n/a{
5545n/a PyObject *last;
5546n/a Py_ssize_t len = Py_SIZE(self->stack);
5547n/a
5548n/a if (len <= self->stack->fence)
5549n/a return Pdata_stack_underflow(self->stack);
5550n/a last = self->stack->data[len - 1];
5551n/a PDATA_APPEND(self->stack, last, -1);
5552n/a return 0;
5553n/a}
5554n/a
5555n/astatic int
5556n/aload_get(UnpicklerObject *self)
5557n/a{
5558n/a PyObject *key, *value;
5559n/a Py_ssize_t idx;
5560n/a Py_ssize_t len;
5561n/a char *s;
5562n/a
5563n/a if ((len = _Unpickler_Readline(self, &s)) < 0)
5564n/a return -1;
5565n/a if (len < 2)
5566n/a return bad_readline();
5567n/a
5568n/a key = PyLong_FromString(s, NULL, 10);
5569n/a if (key == NULL)
5570n/a return -1;
5571n/a idx = PyLong_AsSsize_t(key);
5572n/a if (idx == -1 && PyErr_Occurred()) {
5573n/a Py_DECREF(key);
5574n/a return -1;
5575n/a }
5576n/a
5577n/a value = _Unpickler_MemoGet(self, idx);
5578n/a if (value == NULL) {
5579n/a if (!PyErr_Occurred())
5580n/a PyErr_SetObject(PyExc_KeyError, key);
5581n/a Py_DECREF(key);
5582n/a return -1;
5583n/a }
5584n/a Py_DECREF(key);
5585n/a
5586n/a PDATA_APPEND(self->stack, value, -1);
5587n/a return 0;
5588n/a}
5589n/a
5590n/astatic int
5591n/aload_binget(UnpicklerObject *self)
5592n/a{
5593n/a PyObject *value;
5594n/a Py_ssize_t idx;
5595n/a char *s;
5596n/a
5597n/a if (_Unpickler_Read(self, &s, 1) < 0)
5598n/a return -1;
5599n/a
5600n/a idx = Py_CHARMASK(s[0]);
5601n/a
5602n/a value = _Unpickler_MemoGet(self, idx);
5603n/a if (value == NULL) {
5604n/a PyObject *key = PyLong_FromSsize_t(idx);
5605n/a if (key != NULL) {
5606n/a PyErr_SetObject(PyExc_KeyError, key);
5607n/a Py_DECREF(key);
5608n/a }
5609n/a return -1;
5610n/a }
5611n/a
5612n/a PDATA_APPEND(self->stack, value, -1);
5613n/a return 0;
5614n/a}
5615n/a
5616n/astatic int
5617n/aload_long_binget(UnpicklerObject *self)
5618n/a{
5619n/a PyObject *value;
5620n/a Py_ssize_t idx;
5621n/a char *s;
5622n/a
5623n/a if (_Unpickler_Read(self, &s, 4) < 0)
5624n/a return -1;
5625n/a
5626n/a idx = calc_binsize(s, 4);
5627n/a
5628n/a value = _Unpickler_MemoGet(self, idx);
5629n/a if (value == NULL) {
5630n/a PyObject *key = PyLong_FromSsize_t(idx);
5631n/a if (key != NULL) {
5632n/a PyErr_SetObject(PyExc_KeyError, key);
5633n/a Py_DECREF(key);
5634n/a }
5635n/a return -1;
5636n/a }
5637n/a
5638n/a PDATA_APPEND(self->stack, value, -1);
5639n/a return 0;
5640n/a}
5641n/a
5642n/a/* Push an object from the extension registry (EXT[124]). nbytes is
5643n/a * the number of bytes following the opcode, holding the index (code) value.
5644n/a */
5645n/astatic int
5646n/aload_extension(UnpicklerObject *self, int nbytes)
5647n/a{
5648n/a char *codebytes; /* the nbytes bytes after the opcode */
5649n/a long code; /* calc_binint returns long */
5650n/a PyObject *py_code; /* code as a Python int */
5651n/a PyObject *obj; /* the object to push */
5652n/a PyObject *pair; /* (module_name, class_name) */
5653n/a PyObject *module_name, *class_name;
5654n/a PickleState *st = _Pickle_GetGlobalState();
5655n/a
5656n/a assert(nbytes == 1 || nbytes == 2 || nbytes == 4);
5657n/a if (_Unpickler_Read(self, &codebytes, nbytes) < 0)
5658n/a return -1;
5659n/a code = calc_binint(codebytes, nbytes);
5660n/a if (code <= 0) { /* note that 0 is forbidden */
5661n/a /* Corrupt or hostile pickle. */
5662n/a PyErr_SetString(st->UnpicklingError, "EXT specifies code <= 0");
5663n/a return -1;
5664n/a }
5665n/a
5666n/a /* Look for the code in the cache. */
5667n/a py_code = PyLong_FromLong(code);
5668n/a if (py_code == NULL)
5669n/a return -1;
5670n/a obj = PyDict_GetItemWithError(st->extension_cache, py_code);
5671n/a if (obj != NULL) {
5672n/a /* Bingo. */
5673n/a Py_DECREF(py_code);
5674n/a PDATA_APPEND(self->stack, obj, -1);
5675n/a return 0;
5676n/a }
5677n/a if (PyErr_Occurred()) {
5678n/a Py_DECREF(py_code);
5679n/a return -1;
5680n/a }
5681n/a
5682n/a /* Look up the (module_name, class_name) pair. */
5683n/a pair = PyDict_GetItemWithError(st->inverted_registry, py_code);
5684n/a if (pair == NULL) {
5685n/a Py_DECREF(py_code);
5686n/a if (!PyErr_Occurred()) {
5687n/a PyErr_Format(PyExc_ValueError, "unregistered extension "
5688n/a "code %ld", code);
5689n/a }
5690n/a return -1;
5691n/a }
5692n/a /* Since the extension registry is manipulable via Python code,
5693n/a * confirm that pair is really a 2-tuple of strings.
5694n/a */
5695n/a if (!PyTuple_Check(pair) || PyTuple_Size(pair) != 2 ||
5696n/a !PyUnicode_Check(module_name = PyTuple_GET_ITEM(pair, 0)) ||
5697n/a !PyUnicode_Check(class_name = PyTuple_GET_ITEM(pair, 1))) {
5698n/a Py_DECREF(py_code);
5699n/a PyErr_Format(PyExc_ValueError, "_inverted_registry[%ld] "
5700n/a "isn't a 2-tuple of strings", code);
5701n/a return -1;
5702n/a }
5703n/a /* Load the object. */
5704n/a obj = find_class(self, module_name, class_name);
5705n/a if (obj == NULL) {
5706n/a Py_DECREF(py_code);
5707n/a return -1;
5708n/a }
5709n/a /* Cache code -> obj. */
5710n/a code = PyDict_SetItem(st->extension_cache, py_code, obj);
5711n/a Py_DECREF(py_code);
5712n/a if (code < 0) {
5713n/a Py_DECREF(obj);
5714n/a return -1;
5715n/a }
5716n/a PDATA_PUSH(self->stack, obj, -1);
5717n/a return 0;
5718n/a}
5719n/a
5720n/astatic int
5721n/aload_put(UnpicklerObject *self)
5722n/a{
5723n/a PyObject *key, *value;
5724n/a Py_ssize_t idx;
5725n/a Py_ssize_t len;
5726n/a char *s = NULL;
5727n/a
5728n/a if ((len = _Unpickler_Readline(self, &s)) < 0)
5729n/a return -1;
5730n/a if (len < 2)
5731n/a return bad_readline();
5732n/a if (Py_SIZE(self->stack) <= self->stack->fence)
5733n/a return Pdata_stack_underflow(self->stack);
5734n/a value = self->stack->data[Py_SIZE(self->stack) - 1];
5735n/a
5736n/a key = PyLong_FromString(s, NULL, 10);
5737n/a if (key == NULL)
5738n/a return -1;
5739n/a idx = PyLong_AsSsize_t(key);
5740n/a Py_DECREF(key);
5741n/a if (idx < 0) {
5742n/a if (!PyErr_Occurred())
5743n/a PyErr_SetString(PyExc_ValueError,
5744n/a "negative PUT argument");
5745n/a return -1;
5746n/a }
5747n/a
5748n/a return _Unpickler_MemoPut(self, idx, value);
5749n/a}
5750n/a
5751n/astatic int
5752n/aload_binput(UnpicklerObject *self)
5753n/a{
5754n/a PyObject *value;
5755n/a Py_ssize_t idx;
5756n/a char *s;
5757n/a
5758n/a if (_Unpickler_Read(self, &s, 1) < 0)
5759n/a return -1;
5760n/a
5761n/a if (Py_SIZE(self->stack) <= self->stack->fence)
5762n/a return Pdata_stack_underflow(self->stack);
5763n/a value = self->stack->data[Py_SIZE(self->stack) - 1];
5764n/a
5765n/a idx = Py_CHARMASK(s[0]);
5766n/a
5767n/a return _Unpickler_MemoPut(self, idx, value);
5768n/a}
5769n/a
5770n/astatic int
5771n/aload_long_binput(UnpicklerObject *self)
5772n/a{
5773n/a PyObject *value;
5774n/a Py_ssize_t idx;
5775n/a char *s;
5776n/a
5777n/a if (_Unpickler_Read(self, &s, 4) < 0)
5778n/a return -1;
5779n/a
5780n/a if (Py_SIZE(self->stack) <= self->stack->fence)
5781n/a return Pdata_stack_underflow(self->stack);
5782n/a value = self->stack->data[Py_SIZE(self->stack) - 1];
5783n/a
5784n/a idx = calc_binsize(s, 4);
5785n/a if (idx < 0) {
5786n/a PyErr_SetString(PyExc_ValueError,
5787n/a "negative LONG_BINPUT argument");
5788n/a return -1;
5789n/a }
5790n/a
5791n/a return _Unpickler_MemoPut(self, idx, value);
5792n/a}
5793n/a
5794n/astatic int
5795n/aload_memoize(UnpicklerObject *self)
5796n/a{
5797n/a PyObject *value;
5798n/a
5799n/a if (Py_SIZE(self->stack) <= self->stack->fence)
5800n/a return Pdata_stack_underflow(self->stack);
5801n/a value = self->stack->data[Py_SIZE(self->stack) - 1];
5802n/a
5803n/a return _Unpickler_MemoPut(self, self->memo_len, value);
5804n/a}
5805n/a
5806n/astatic int
5807n/ado_append(UnpicklerObject *self, Py_ssize_t x)
5808n/a{
5809n/a PyObject *value;
5810n/a PyObject *slice;
5811n/a PyObject *list;
5812n/a PyObject *result;
5813n/a Py_ssize_t len, i;
5814n/a
5815n/a len = Py_SIZE(self->stack);
5816n/a if (x > len || x <= self->stack->fence)
5817n/a return Pdata_stack_underflow(self->stack);
5818n/a if (len == x) /* nothing to do */
5819n/a return 0;
5820n/a
5821n/a list = self->stack->data[x - 1];
5822n/a
5823n/a if (PyList_CheckExact(list)) {
5824n/a Py_ssize_t list_len;
5825n/a int ret;
5826n/a
5827n/a slice = Pdata_poplist(self->stack, x);
5828n/a if (!slice)
5829n/a return -1;
5830n/a list_len = PyList_GET_SIZE(list);
5831n/a ret = PyList_SetSlice(list, list_len, list_len, slice);
5832n/a Py_DECREF(slice);
5833n/a return ret;
5834n/a }
5835n/a else {
5836n/a PyObject *extend_func;
5837n/a _Py_IDENTIFIER(extend);
5838n/a
5839n/a extend_func = _PyObject_GetAttrId(list, &PyId_extend);
5840n/a if (extend_func != NULL) {
5841n/a slice = Pdata_poplist(self->stack, x);
5842n/a if (!slice) {
5843n/a Py_DECREF(extend_func);
5844n/a return -1;
5845n/a }
5846n/a result = _Pickle_FastCall(extend_func, slice);
5847n/a Py_DECREF(extend_func);
5848n/a if (result == NULL)
5849n/a return -1;
5850n/a Py_DECREF(result);
5851n/a }
5852n/a else {
5853n/a PyObject *append_func;
5854n/a _Py_IDENTIFIER(append);
5855n/a
5856n/a /* Even if the PEP 307 requires extend() and append() methods,
5857n/a fall back on append() if the object has no extend() method
5858n/a for backward compatibility. */
5859n/a PyErr_Clear();
5860n/a append_func = _PyObject_GetAttrId(list, &PyId_append);
5861n/a if (append_func == NULL)
5862n/a return -1;
5863n/a for (i = x; i < len; i++) {
5864n/a value = self->stack->data[i];
5865n/a result = _Pickle_FastCall(append_func, value);
5866n/a if (result == NULL) {
5867n/a Pdata_clear(self->stack, i + 1);
5868n/a Py_SIZE(self->stack) = x;
5869n/a Py_DECREF(append_func);
5870n/a return -1;
5871n/a }
5872n/a Py_DECREF(result);
5873n/a }
5874n/a Py_SIZE(self->stack) = x;
5875n/a Py_DECREF(append_func);
5876n/a }
5877n/a }
5878n/a
5879n/a return 0;
5880n/a}
5881n/a
5882n/astatic int
5883n/aload_append(UnpicklerObject *self)
5884n/a{
5885n/a if (Py_SIZE(self->stack) - 1 <= self->stack->fence)
5886n/a return Pdata_stack_underflow(self->stack);
5887n/a return do_append(self, Py_SIZE(self->stack) - 1);
5888n/a}
5889n/a
5890n/astatic int
5891n/aload_appends(UnpicklerObject *self)
5892n/a{
5893n/a Py_ssize_t i = marker(self);
5894n/a if (i < 0)
5895n/a return -1;
5896n/a return do_append(self, i);
5897n/a}
5898n/a
5899n/astatic int
5900n/ado_setitems(UnpicklerObject *self, Py_ssize_t x)
5901n/a{
5902n/a PyObject *value, *key;
5903n/a PyObject *dict;
5904n/a Py_ssize_t len, i;
5905n/a int status = 0;
5906n/a
5907n/a len = Py_SIZE(self->stack);
5908n/a if (x > len || x <= self->stack->fence)
5909n/a return Pdata_stack_underflow(self->stack);
5910n/a if (len == x) /* nothing to do */
5911n/a return 0;
5912n/a if ((len - x) % 2 != 0) {
5913n/a PickleState *st = _Pickle_GetGlobalState();
5914n/a /* Currupt or hostile pickle -- we never write one like this. */
5915n/a PyErr_SetString(st->UnpicklingError,
5916n/a "odd number of items for SETITEMS");
5917n/a return -1;
5918n/a }
5919n/a
5920n/a /* Here, dict does not actually need to be a PyDict; it could be anything
5921n/a that supports the __setitem__ attribute. */
5922n/a dict = self->stack->data[x - 1];
5923n/a
5924n/a for (i = x + 1; i < len; i += 2) {
5925n/a key = self->stack->data[i - 1];
5926n/a value = self->stack->data[i];
5927n/a if (PyObject_SetItem(dict, key, value) < 0) {
5928n/a status = -1;
5929n/a break;
5930n/a }
5931n/a }
5932n/a
5933n/a Pdata_clear(self->stack, x);
5934n/a return status;
5935n/a}
5936n/a
5937n/astatic int
5938n/aload_setitem(UnpicklerObject *self)
5939n/a{
5940n/a return do_setitems(self, Py_SIZE(self->stack) - 2);
5941n/a}
5942n/a
5943n/astatic int
5944n/aload_setitems(UnpicklerObject *self)
5945n/a{
5946n/a Py_ssize_t i = marker(self);
5947n/a if (i < 0)
5948n/a return -1;
5949n/a return do_setitems(self, i);
5950n/a}
5951n/a
5952n/astatic int
5953n/aload_additems(UnpicklerObject *self)
5954n/a{
5955n/a PyObject *set;
5956n/a Py_ssize_t mark, len, i;
5957n/a
5958n/a mark = marker(self);
5959n/a if (mark < 0)
5960n/a return -1;
5961n/a len = Py_SIZE(self->stack);
5962n/a if (mark > len || mark <= self->stack->fence)
5963n/a return Pdata_stack_underflow(self->stack);
5964n/a if (len == mark) /* nothing to do */
5965n/a return 0;
5966n/a
5967n/a set = self->stack->data[mark - 1];
5968n/a
5969n/a if (PySet_Check(set)) {
5970n/a PyObject *items;
5971n/a int status;
5972n/a
5973n/a items = Pdata_poptuple(self->stack, mark);
5974n/a if (items == NULL)
5975n/a return -1;
5976n/a
5977n/a status = _PySet_Update(set, items);
5978n/a Py_DECREF(items);
5979n/a return status;
5980n/a }
5981n/a else {
5982n/a PyObject *add_func;
5983n/a _Py_IDENTIFIER(add);
5984n/a
5985n/a add_func = _PyObject_GetAttrId(set, &PyId_add);
5986n/a if (add_func == NULL)
5987n/a return -1;
5988n/a for (i = mark; i < len; i++) {
5989n/a PyObject *result;
5990n/a PyObject *item;
5991n/a
5992n/a item = self->stack->data[i];
5993n/a result = _Pickle_FastCall(add_func, item);
5994n/a if (result == NULL) {
5995n/a Pdata_clear(self->stack, i + 1);
5996n/a Py_SIZE(self->stack) = mark;
5997n/a return -1;
5998n/a }
5999n/a Py_DECREF(result);
6000n/a }
6001n/a Py_SIZE(self->stack) = mark;
6002n/a }
6003n/a
6004n/a return 0;
6005n/a}
6006n/a
6007n/astatic int
6008n/aload_build(UnpicklerObject *self)
6009n/a{
6010n/a PyObject *state, *inst, *slotstate;
6011n/a PyObject *setstate;
6012n/a int status = 0;
6013n/a _Py_IDENTIFIER(__setstate__);
6014n/a
6015n/a /* Stack is ... instance, state. We want to leave instance at
6016n/a * the stack top, possibly mutated via instance.__setstate__(state).
6017n/a */
6018n/a if (Py_SIZE(self->stack) - 2 < self->stack->fence)
6019n/a return Pdata_stack_underflow(self->stack);
6020n/a
6021n/a PDATA_POP(self->stack, state);
6022n/a if (state == NULL)
6023n/a return -1;
6024n/a
6025n/a inst = self->stack->data[Py_SIZE(self->stack) - 1];
6026n/a
6027n/a setstate = _PyObject_GetAttrId(inst, &PyId___setstate__);
6028n/a if (setstate == NULL) {
6029n/a if (PyErr_ExceptionMatches(PyExc_AttributeError))
6030n/a PyErr_Clear();
6031n/a else {
6032n/a Py_DECREF(state);
6033n/a return -1;
6034n/a }
6035n/a }
6036n/a else {
6037n/a PyObject *result;
6038n/a
6039n/a /* The explicit __setstate__ is responsible for everything. */
6040n/a result = _Pickle_FastCall(setstate, state);
6041n/a Py_DECREF(setstate);
6042n/a if (result == NULL)
6043n/a return -1;
6044n/a Py_DECREF(result);
6045n/a return 0;
6046n/a }
6047n/a
6048n/a /* A default __setstate__. First see whether state embeds a
6049n/a * slot state dict too (a proto 2 addition).
6050n/a */
6051n/a if (PyTuple_Check(state) && Py_SIZE(state) == 2) {
6052n/a PyObject *tmp = state;
6053n/a
6054n/a state = PyTuple_GET_ITEM(tmp, 0);
6055n/a slotstate = PyTuple_GET_ITEM(tmp, 1);
6056n/a Py_INCREF(state);
6057n/a Py_INCREF(slotstate);
6058n/a Py_DECREF(tmp);
6059n/a }
6060n/a else
6061n/a slotstate = NULL;
6062n/a
6063n/a /* Set inst.__dict__ from the state dict (if any). */
6064n/a if (state != Py_None) {
6065n/a PyObject *dict;
6066n/a PyObject *d_key, *d_value;
6067n/a Py_ssize_t i;
6068n/a _Py_IDENTIFIER(__dict__);
6069n/a
6070n/a if (!PyDict_Check(state)) {
6071n/a PickleState *st = _Pickle_GetGlobalState();
6072n/a PyErr_SetString(st->UnpicklingError, "state is not a dictionary");
6073n/a goto error;
6074n/a }
6075n/a dict = _PyObject_GetAttrId(inst, &PyId___dict__);
6076n/a if (dict == NULL)
6077n/a goto error;
6078n/a
6079n/a i = 0;
6080n/a while (PyDict_Next(state, &i, &d_key, &d_value)) {
6081n/a /* normally the keys for instance attributes are
6082n/a interned. we should try to do that here. */
6083n/a Py_INCREF(d_key);
6084n/a if (PyUnicode_CheckExact(d_key))
6085n/a PyUnicode_InternInPlace(&d_key);
6086n/a if (PyObject_SetItem(dict, d_key, d_value) < 0) {
6087n/a Py_DECREF(d_key);
6088n/a goto error;
6089n/a }
6090n/a Py_DECREF(d_key);
6091n/a }
6092n/a Py_DECREF(dict);
6093n/a }
6094n/a
6095n/a /* Also set instance attributes from the slotstate dict (if any). */
6096n/a if (slotstate != NULL) {
6097n/a PyObject *d_key, *d_value;
6098n/a Py_ssize_t i;
6099n/a
6100n/a if (!PyDict_Check(slotstate)) {
6101n/a PickleState *st = _Pickle_GetGlobalState();
6102n/a PyErr_SetString(st->UnpicklingError,
6103n/a "slot state is not a dictionary");
6104n/a goto error;
6105n/a }
6106n/a i = 0;
6107n/a while (PyDict_Next(slotstate, &i, &d_key, &d_value)) {
6108n/a if (PyObject_SetAttr(inst, d_key, d_value) < 0)
6109n/a goto error;
6110n/a }
6111n/a }
6112n/a
6113n/a if (0) {
6114n/a error:
6115n/a status = -1;
6116n/a }
6117n/a
6118n/a Py_DECREF(state);
6119n/a Py_XDECREF(slotstate);
6120n/a return status;
6121n/a}
6122n/a
6123n/astatic int
6124n/aload_mark(UnpicklerObject *self)
6125n/a{
6126n/a
6127n/a /* Note that we split the (pickle.py) stack into two stacks, an
6128n/a * object stack and a mark stack. Here we push a mark onto the
6129n/a * mark stack.
6130n/a */
6131n/a
6132n/a if ((self->num_marks + 1) >= self->marks_size) {
6133n/a size_t alloc;
6134n/a
6135n/a /* Use the size_t type to check for overflow. */
6136n/a alloc = ((size_t)self->num_marks << 1) + 20;
6137n/a if (alloc > (PY_SSIZE_T_MAX / sizeof(Py_ssize_t)) ||
6138n/a alloc <= ((size_t)self->num_marks + 1)) {
6139n/a PyErr_NoMemory();
6140n/a return -1;
6141n/a }
6142n/a
6143n/a if (self->marks == NULL)
6144n/a self->marks = PyMem_NEW(Py_ssize_t, alloc);
6145n/a else
6146n/a PyMem_RESIZE(self->marks, Py_ssize_t, alloc);
6147n/a if (self->marks == NULL) {
6148n/a self->marks_size = 0;
6149n/a PyErr_NoMemory();
6150n/a return -1;
6151n/a }
6152n/a self->marks_size = (Py_ssize_t)alloc;
6153n/a }
6154n/a
6155n/a self->stack->mark_set = 1;
6156n/a self->marks[self->num_marks++] = self->stack->fence = Py_SIZE(self->stack);
6157n/a
6158n/a return 0;
6159n/a}
6160n/a
6161n/astatic int
6162n/aload_reduce(UnpicklerObject *self)
6163n/a{
6164n/a PyObject *callable = NULL;
6165n/a PyObject *argtup = NULL;
6166n/a PyObject *obj = NULL;
6167n/a
6168n/a PDATA_POP(self->stack, argtup);
6169n/a if (argtup == NULL)
6170n/a return -1;
6171n/a PDATA_POP(self->stack, callable);
6172n/a if (callable) {
6173n/a obj = PyObject_CallObject(callable, argtup);
6174n/a Py_DECREF(callable);
6175n/a }
6176n/a Py_DECREF(argtup);
6177n/a
6178n/a if (obj == NULL)
6179n/a return -1;
6180n/a
6181n/a PDATA_PUSH(self->stack, obj, -1);
6182n/a return 0;
6183n/a}
6184n/a
6185n/a/* Just raises an error if we don't know the protocol specified. PROTO
6186n/a * is the first opcode for protocols >= 2.
6187n/a */
6188n/astatic int
6189n/aload_proto(UnpicklerObject *self)
6190n/a{
6191n/a char *s;
6192n/a int i;
6193n/a
6194n/a if (_Unpickler_Read(self, &s, 1) < 0)
6195n/a return -1;
6196n/a
6197n/a i = (unsigned char)s[0];
6198n/a if (i <= HIGHEST_PROTOCOL) {
6199n/a self->proto = i;
6200n/a return 0;
6201n/a }
6202n/a
6203n/a PyErr_Format(PyExc_ValueError, "unsupported pickle protocol: %d", i);
6204n/a return -1;
6205n/a}
6206n/a
6207n/astatic int
6208n/aload_frame(UnpicklerObject *self)
6209n/a{
6210n/a char *s;
6211n/a Py_ssize_t frame_len;
6212n/a
6213n/a if (_Unpickler_Read(self, &s, 8) < 0)
6214n/a return -1;
6215n/a
6216n/a frame_len = calc_binsize(s, 8);
6217n/a if (frame_len < 0) {
6218n/a PyErr_Format(PyExc_OverflowError,
6219n/a "FRAME length exceeds system's maximum of %zd bytes",
6220n/a PY_SSIZE_T_MAX);
6221n/a return -1;
6222n/a }
6223n/a
6224n/a if (_Unpickler_Read(self, &s, frame_len) < 0)
6225n/a return -1;
6226n/a
6227n/a /* Rewind to start of frame */
6228n/a self->next_read_idx -= frame_len;
6229n/a return 0;
6230n/a}
6231n/a
6232n/astatic PyObject *
6233n/aload(UnpicklerObject *self)
6234n/a{
6235n/a PyObject *value = NULL;
6236n/a char *s = NULL;
6237n/a
6238n/a self->num_marks = 0;
6239n/a self->stack->mark_set = 0;
6240n/a self->stack->fence = 0;
6241n/a self->proto = 0;
6242n/a if (Py_SIZE(self->stack))
6243n/a Pdata_clear(self->stack, 0);
6244n/a
6245n/a /* Convenient macros for the dispatch while-switch loop just below. */
6246n/a#define OP(opcode, load_func) \
6247n/a case opcode: if (load_func(self) < 0) break; continue;
6248n/a
6249n/a#define OP_ARG(opcode, load_func, arg) \
6250n/a case opcode: if (load_func(self, (arg)) < 0) break; continue;
6251n/a
6252n/a while (1) {
6253n/a if (_Unpickler_Read(self, &s, 1) < 0) {
6254n/a PickleState *st = _Pickle_GetGlobalState();
6255n/a if (PyErr_ExceptionMatches(st->UnpicklingError)) {
6256n/a PyErr_Format(PyExc_EOFError, "Ran out of input");
6257n/a }
6258n/a return NULL;
6259n/a }
6260n/a
6261n/a switch ((enum opcode)s[0]) {
6262n/a OP(NONE, load_none)
6263n/a OP(BININT, load_binint)
6264n/a OP(BININT1, load_binint1)
6265n/a OP(BININT2, load_binint2)
6266n/a OP(INT, load_int)
6267n/a OP(LONG, load_long)
6268n/a OP_ARG(LONG1, load_counted_long, 1)
6269n/a OP_ARG(LONG4, load_counted_long, 4)
6270n/a OP(FLOAT, load_float)
6271n/a OP(BINFLOAT, load_binfloat)
6272n/a OP_ARG(SHORT_BINBYTES, load_counted_binbytes, 1)
6273n/a OP_ARG(BINBYTES, load_counted_binbytes, 4)
6274n/a OP_ARG(BINBYTES8, load_counted_binbytes, 8)
6275n/a OP_ARG(SHORT_BINSTRING, load_counted_binstring, 1)
6276n/a OP_ARG(BINSTRING, load_counted_binstring, 4)
6277n/a OP(STRING, load_string)
6278n/a OP(UNICODE, load_unicode)
6279n/a OP_ARG(SHORT_BINUNICODE, load_counted_binunicode, 1)
6280n/a OP_ARG(BINUNICODE, load_counted_binunicode, 4)
6281n/a OP_ARG(BINUNICODE8, load_counted_binunicode, 8)
6282n/a OP_ARG(EMPTY_TUPLE, load_counted_tuple, 0)
6283n/a OP_ARG(TUPLE1, load_counted_tuple, 1)
6284n/a OP_ARG(TUPLE2, load_counted_tuple, 2)
6285n/a OP_ARG(TUPLE3, load_counted_tuple, 3)
6286n/a OP(TUPLE, load_tuple)
6287n/a OP(EMPTY_LIST, load_empty_list)
6288n/a OP(LIST, load_list)
6289n/a OP(EMPTY_DICT, load_empty_dict)
6290n/a OP(DICT, load_dict)
6291n/a OP(EMPTY_SET, load_empty_set)
6292n/a OP(ADDITEMS, load_additems)
6293n/a OP(FROZENSET, load_frozenset)
6294n/a OP(OBJ, load_obj)
6295n/a OP(INST, load_inst)
6296n/a OP(NEWOBJ, load_newobj)
6297n/a OP(NEWOBJ_EX, load_newobj_ex)
6298n/a OP(GLOBAL, load_global)
6299n/a OP(STACK_GLOBAL, load_stack_global)
6300n/a OP(APPEND, load_append)
6301n/a OP(APPENDS, load_appends)
6302n/a OP(BUILD, load_build)
6303n/a OP(DUP, load_dup)
6304n/a OP(BINGET, load_binget)
6305n/a OP(LONG_BINGET, load_long_binget)
6306n/a OP(GET, load_get)
6307n/a OP(MARK, load_mark)
6308n/a OP(BINPUT, load_binput)
6309n/a OP(LONG_BINPUT, load_long_binput)
6310n/a OP(PUT, load_put)
6311n/a OP(MEMOIZE, load_memoize)
6312n/a OP(POP, load_pop)
6313n/a OP(POP_MARK, load_pop_mark)
6314n/a OP(SETITEM, load_setitem)
6315n/a OP(SETITEMS, load_setitems)
6316n/a OP(PERSID, load_persid)
6317n/a OP(BINPERSID, load_binpersid)
6318n/a OP(REDUCE, load_reduce)
6319n/a OP(PROTO, load_proto)
6320n/a OP(FRAME, load_frame)
6321n/a OP_ARG(EXT1, load_extension, 1)
6322n/a OP_ARG(EXT2, load_extension, 2)
6323n/a OP_ARG(EXT4, load_extension, 4)
6324n/a OP_ARG(NEWTRUE, load_bool, Py_True)
6325n/a OP_ARG(NEWFALSE, load_bool, Py_False)
6326n/a
6327n/a case STOP:
6328n/a break;
6329n/a
6330n/a default:
6331n/a {
6332n/a PickleState *st = _Pickle_GetGlobalState();
6333n/a unsigned char c = (unsigned char) *s;
6334n/a if (0x20 <= c && c <= 0x7e && c != '\'' && c != '\\') {
6335n/a PyErr_Format(st->UnpicklingError,
6336n/a "invalid load key, '%c'.", c);
6337n/a }
6338n/a else {
6339n/a PyErr_Format(st->UnpicklingError,
6340n/a "invalid load key, '\\x%02x'.", c);
6341n/a }
6342n/a return NULL;
6343n/a }
6344n/a }
6345n/a
6346n/a break; /* and we are done! */
6347n/a }
6348n/a
6349n/a if (PyErr_Occurred()) {
6350n/a return NULL;
6351n/a }
6352n/a
6353n/a if (_Unpickler_SkipConsumed(self) < 0)
6354n/a return NULL;
6355n/a
6356n/a PDATA_POP(self->stack, value);
6357n/a return value;
6358n/a}
6359n/a
6360n/a/*[clinic input]
6361n/a
6362n/a_pickle.Unpickler.load
6363n/a
6364n/aLoad a pickle.
6365n/a
6366n/aRead a pickled object representation from the open file object given
6367n/ain the constructor, and return the reconstituted object hierarchy
6368n/aspecified therein.
6369n/a[clinic start generated code]*/
6370n/a
6371n/astatic PyObject *
6372n/a_pickle_Unpickler_load_impl(UnpicklerObject *self)
6373n/a/*[clinic end generated code: output=fdcc488aad675b14 input=acbb91a42fa9b7b9]*/
6374n/a{
6375n/a UnpicklerObject *unpickler = (UnpicklerObject*)self;
6376n/a
6377n/a /* Check whether the Unpickler was initialized correctly. This prevents
6378n/a segfaulting if a subclass overridden __init__ with a function that does
6379n/a not call Unpickler.__init__(). Here, we simply ensure that self->read
6380n/a is not NULL. */
6381n/a if (unpickler->read == NULL) {
6382n/a PickleState *st = _Pickle_GetGlobalState();
6383n/a PyErr_Format(st->UnpicklingError,
6384n/a "Unpickler.__init__() was not called by %s.__init__()",
6385n/a Py_TYPE(unpickler)->tp_name);
6386n/a return NULL;
6387n/a }
6388n/a
6389n/a return load(unpickler);
6390n/a}
6391n/a
6392n/a/* The name of find_class() is misleading. In newer pickle protocols, this
6393n/a function is used for loading any global (i.e., functions), not just
6394n/a classes. The name is kept only for backward compatibility. */
6395n/a
6396n/a/*[clinic input]
6397n/a
6398n/a_pickle.Unpickler.find_class
6399n/a
6400n/a module_name: object
6401n/a global_name: object
6402n/a /
6403n/a
6404n/aReturn an object from a specified module.
6405n/a
6406n/aIf necessary, the module will be imported. Subclasses may override
6407n/athis method (e.g. to restrict unpickling of arbitrary classes and
6408n/afunctions).
6409n/a
6410n/aThis method is called whenever a class or a function object is
6411n/aneeded. Both arguments passed are str objects.
6412n/a[clinic start generated code]*/
6413n/a
6414n/astatic PyObject *
6415n/a_pickle_Unpickler_find_class_impl(UnpicklerObject *self,
6416n/a PyObject *module_name,
6417n/a PyObject *global_name)
6418n/a/*[clinic end generated code: output=becc08d7f9ed41e3 input=e2e6a865de093ef4]*/
6419n/a{
6420n/a PyObject *global;
6421n/a PyObject *modules_dict;
6422n/a PyObject *module;
6423n/a _Py_IDENTIFIER(modules);
6424n/a
6425n/a /* Try to map the old names used in Python 2.x to the new ones used in
6426n/a Python 3.x. We do this only with old pickle protocols and when the
6427n/a user has not disabled the feature. */
6428n/a if (self->proto < 3 && self->fix_imports) {
6429n/a PyObject *key;
6430n/a PyObject *item;
6431n/a PickleState *st = _Pickle_GetGlobalState();
6432n/a
6433n/a /* Check if the global (i.e., a function or a class) was renamed
6434n/a or moved to another module. */
6435n/a key = PyTuple_Pack(2, module_name, global_name);
6436n/a if (key == NULL)
6437n/a return NULL;
6438n/a item = PyDict_GetItemWithError(st->name_mapping_2to3, key);
6439n/a Py_DECREF(key);
6440n/a if (item) {
6441n/a if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) {
6442n/a PyErr_Format(PyExc_RuntimeError,
6443n/a "_compat_pickle.NAME_MAPPING values should be "
6444n/a "2-tuples, not %.200s", Py_TYPE(item)->tp_name);
6445n/a return NULL;
6446n/a }
6447n/a module_name = PyTuple_GET_ITEM(item, 0);
6448n/a global_name = PyTuple_GET_ITEM(item, 1);
6449n/a if (!PyUnicode_Check(module_name) ||
6450n/a !PyUnicode_Check(global_name)) {
6451n/a PyErr_Format(PyExc_RuntimeError,
6452n/a "_compat_pickle.NAME_MAPPING values should be "
6453n/a "pairs of str, not (%.200s, %.200s)",
6454n/a Py_TYPE(module_name)->tp_name,
6455n/a Py_TYPE(global_name)->tp_name);
6456n/a return NULL;
6457n/a }
6458n/a }
6459n/a else if (PyErr_Occurred()) {
6460n/a return NULL;
6461n/a }
6462n/a else {
6463n/a /* Check if the module was renamed. */
6464n/a item = PyDict_GetItemWithError(st->import_mapping_2to3, module_name);
6465n/a if (item) {
6466n/a if (!PyUnicode_Check(item)) {
6467n/a PyErr_Format(PyExc_RuntimeError,
6468n/a "_compat_pickle.IMPORT_MAPPING values should be "
6469n/a "strings, not %.200s", Py_TYPE(item)->tp_name);
6470n/a return NULL;
6471n/a }
6472n/a module_name = item;
6473n/a }
6474n/a else if (PyErr_Occurred()) {
6475n/a return NULL;
6476n/a }
6477n/a }
6478n/a }
6479n/a
6480n/a modules_dict = _PySys_GetObjectId(&PyId_modules);
6481n/a if (modules_dict == NULL) {
6482n/a PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules");
6483n/a return NULL;
6484n/a }
6485n/a
6486n/a module = PyDict_GetItemWithError(modules_dict, module_name);
6487n/a if (module == NULL) {
6488n/a if (PyErr_Occurred())
6489n/a return NULL;
6490n/a module = PyImport_Import(module_name);
6491n/a if (module == NULL)
6492n/a return NULL;
6493n/a global = getattribute(module, global_name, self->proto >= 4);
6494n/a Py_DECREF(module);
6495n/a }
6496n/a else {
6497n/a global = getattribute(module, global_name, self->proto >= 4);
6498n/a }
6499n/a return global;
6500n/a}
6501n/a
6502n/a/*[clinic input]
6503n/a
6504n/a_pickle.Unpickler.__sizeof__ -> Py_ssize_t
6505n/a
6506n/aReturns size in memory, in bytes.
6507n/a[clinic start generated code]*/
6508n/a
6509n/astatic Py_ssize_t
6510n/a_pickle_Unpickler___sizeof___impl(UnpicklerObject *self)
6511n/a/*[clinic end generated code: output=119d9d03ad4c7651 input=13333471fdeedf5e]*/
6512n/a{
6513n/a Py_ssize_t res;
6514n/a
6515n/a res = _PyObject_SIZE(Py_TYPE(self));
6516n/a if (self->memo != NULL)
6517n/a res += self->memo_size * sizeof(PyObject *);
6518n/a if (self->marks != NULL)
6519n/a res += self->marks_size * sizeof(Py_ssize_t);
6520n/a if (self->input_line != NULL)
6521n/a res += strlen(self->input_line) + 1;
6522n/a if (self->encoding != NULL)
6523n/a res += strlen(self->encoding) + 1;
6524n/a if (self->errors != NULL)
6525n/a res += strlen(self->errors) + 1;
6526n/a return res;
6527n/a}
6528n/a
6529n/astatic struct PyMethodDef Unpickler_methods[] = {
6530n/a _PICKLE_UNPICKLER_LOAD_METHODDEF
6531n/a _PICKLE_UNPICKLER_FIND_CLASS_METHODDEF
6532n/a _PICKLE_UNPICKLER___SIZEOF___METHODDEF
6533n/a {NULL, NULL} /* sentinel */
6534n/a};
6535n/a
6536n/astatic void
6537n/aUnpickler_dealloc(UnpicklerObject *self)
6538n/a{
6539n/a PyObject_GC_UnTrack((PyObject *)self);
6540n/a Py_XDECREF(self->readline);
6541n/a Py_XDECREF(self->read);
6542n/a Py_XDECREF(self->peek);
6543n/a Py_XDECREF(self->stack);
6544n/a Py_XDECREF(self->pers_func);
6545n/a if (self->buffer.buf != NULL) {
6546n/a PyBuffer_Release(&self->buffer);
6547n/a self->buffer.buf = NULL;
6548n/a }
6549n/a
6550n/a _Unpickler_MemoCleanup(self);
6551n/a PyMem_Free(self->marks);
6552n/a PyMem_Free(self->input_line);
6553n/a PyMem_Free(self->encoding);
6554n/a PyMem_Free(self->errors);
6555n/a
6556n/a Py_TYPE(self)->tp_free((PyObject *)self);
6557n/a}
6558n/a
6559n/astatic int
6560n/aUnpickler_traverse(UnpicklerObject *self, visitproc visit, void *arg)
6561n/a{
6562n/a Py_VISIT(self->readline);
6563n/a Py_VISIT(self->read);
6564n/a Py_VISIT(self->peek);
6565n/a Py_VISIT(self->stack);
6566n/a Py_VISIT(self->pers_func);
6567n/a return 0;
6568n/a}
6569n/a
6570n/astatic int
6571n/aUnpickler_clear(UnpicklerObject *self)
6572n/a{
6573n/a Py_CLEAR(self->readline);
6574n/a Py_CLEAR(self->read);
6575n/a Py_CLEAR(self->peek);
6576n/a Py_CLEAR(self->stack);
6577n/a Py_CLEAR(self->pers_func);
6578n/a if (self->buffer.buf != NULL) {
6579n/a PyBuffer_Release(&self->buffer);
6580n/a self->buffer.buf = NULL;
6581n/a }
6582n/a
6583n/a _Unpickler_MemoCleanup(self);
6584n/a PyMem_Free(self->marks);
6585n/a self->marks = NULL;
6586n/a PyMem_Free(self->input_line);
6587n/a self->input_line = NULL;
6588n/a PyMem_Free(self->encoding);
6589n/a self->encoding = NULL;
6590n/a PyMem_Free(self->errors);
6591n/a self->errors = NULL;
6592n/a
6593n/a return 0;
6594n/a}
6595n/a
6596n/a/*[clinic input]
6597n/a
6598n/a_pickle.Unpickler.__init__
6599n/a
6600n/a file: object
6601n/a *
6602n/a fix_imports: bool = True
6603n/a encoding: str = 'ASCII'
6604n/a errors: str = 'strict'
6605n/a
6606n/aThis takes a binary file for reading a pickle data stream.
6607n/a
6608n/aThe protocol version of the pickle is detected automatically, so no
6609n/aprotocol argument is needed. Bytes past the pickled object's
6610n/arepresentation are ignored.
6611n/a
6612n/aThe argument *file* must have two methods, a read() method that takes
6613n/aan integer argument, and a readline() method that requires no
6614n/aarguments. Both methods should return bytes. Thus *file* can be a
6615n/abinary file object opened for reading, an io.BytesIO object, or any
6616n/aother custom object that meets this interface.
6617n/a
6618n/aOptional keyword arguments are *fix_imports*, *encoding* and *errors*,
6619n/awhich are used to control compatibility support for pickle stream
6620n/agenerated by Python 2. If *fix_imports* is True, pickle will try to
6621n/amap the old Python 2 names to the new names used in Python 3. The
6622n/a*encoding* and *errors* tell pickle how to decode 8-bit string
6623n/ainstances pickled by Python 2; these default to 'ASCII' and 'strict',
6624n/arespectively. The *encoding* can be 'bytes' to read these 8-bit
6625n/astring instances as bytes objects.
6626n/a[clinic start generated code]*/
6627n/a
6628n/astatic int
6629n/a_pickle_Unpickler___init___impl(UnpicklerObject *self, PyObject *file,
6630n/a int fix_imports, const char *encoding,
6631n/a const char *errors)
6632n/a/*[clinic end generated code: output=e2c8ce748edc57b0 input=f9b7da04f5f4f335]*/
6633n/a{
6634n/a _Py_IDENTIFIER(persistent_load);
6635n/a
6636n/a /* In case of multiple __init__() calls, clear previous content. */
6637n/a if (self->read != NULL)
6638n/a (void)Unpickler_clear(self);
6639n/a
6640n/a if (_Unpickler_SetInputStream(self, file) < 0)
6641n/a return -1;
6642n/a
6643n/a if (_Unpickler_SetInputEncoding(self, encoding, errors) < 0)
6644n/a return -1;
6645n/a
6646n/a self->fix_imports = fix_imports;
6647n/a if (self->fix_imports == -1)
6648n/a return -1;
6649n/a
6650n/a if (_PyObject_HasAttrId((PyObject *)self, &PyId_persistent_load)) {
6651n/a self->pers_func = _PyObject_GetAttrId((PyObject *)self,
6652n/a &PyId_persistent_load);
6653n/a if (self->pers_func == NULL)
6654n/a return 1;
6655n/a }
6656n/a else {
6657n/a self->pers_func = NULL;
6658n/a }
6659n/a
6660n/a self->stack = (Pdata *)Pdata_New();
6661n/a if (self->stack == NULL)
6662n/a return 1;
6663n/a
6664n/a self->memo_size = 32;
6665n/a self->memo = _Unpickler_NewMemo(self->memo_size);
6666n/a if (self->memo == NULL)
6667n/a return -1;
6668n/a
6669n/a self->proto = 0;
6670n/a
6671n/a return 0;
6672n/a}
6673n/a
6674n/a
6675n/a/* Define a proxy object for the Unpickler's internal memo object. This is to
6676n/a * avoid breaking code like:
6677n/a * unpickler.memo.clear()
6678n/a * and
6679n/a * unpickler.memo = saved_memo
6680n/a * Is this a good idea? Not really, but we don't want to break code that uses
6681n/a * it. Note that we don't implement the entire mapping API here. This is
6682n/a * intentional, as these should be treated as black-box implementation details.
6683n/a *
6684n/a * We do, however, have to implement pickling/unpickling support because of
6685n/a * real-world code like cvs2svn.
6686n/a */
6687n/a
6688n/a/*[clinic input]
6689n/a_pickle.UnpicklerMemoProxy.clear
6690n/a
6691n/aRemove all items from memo.
6692n/a[clinic start generated code]*/
6693n/a
6694n/astatic PyObject *
6695n/a_pickle_UnpicklerMemoProxy_clear_impl(UnpicklerMemoProxyObject *self)
6696n/a/*[clinic end generated code: output=d20cd43f4ba1fb1f input=b1df7c52e7afd9bd]*/
6697n/a{
6698n/a _Unpickler_MemoCleanup(self->unpickler);
6699n/a self->unpickler->memo = _Unpickler_NewMemo(self->unpickler->memo_size);
6700n/a if (self->unpickler->memo == NULL)
6701n/a return NULL;
6702n/a Py_RETURN_NONE;
6703n/a}
6704n/a
6705n/a/*[clinic input]
6706n/a_pickle.UnpicklerMemoProxy.copy
6707n/a
6708n/aCopy the memo to a new object.
6709n/a[clinic start generated code]*/
6710n/a
6711n/astatic PyObject *
6712n/a_pickle_UnpicklerMemoProxy_copy_impl(UnpicklerMemoProxyObject *self)
6713n/a/*[clinic end generated code: output=e12af7e9bc1e4c77 input=97769247ce032c1d]*/
6714n/a{
6715n/a Py_ssize_t i;
6716n/a PyObject *new_memo = PyDict_New();
6717n/a if (new_memo == NULL)
6718n/a return NULL;
6719n/a
6720n/a for (i = 0; i < self->unpickler->memo_size; i++) {
6721n/a int status;
6722n/a PyObject *key, *value;
6723n/a
6724n/a value = self->unpickler->memo[i];
6725n/a if (value == NULL)
6726n/a continue;
6727n/a
6728n/a key = PyLong_FromSsize_t(i);
6729n/a if (key == NULL)
6730n/a goto error;
6731n/a status = PyDict_SetItem(new_memo, key, value);
6732n/a Py_DECREF(key);
6733n/a if (status < 0)
6734n/a goto error;
6735n/a }
6736n/a return new_memo;
6737n/a
6738n/aerror:
6739n/a Py_DECREF(new_memo);
6740n/a return NULL;
6741n/a}
6742n/a
6743n/a/*[clinic input]
6744n/a_pickle.UnpicklerMemoProxy.__reduce__
6745n/a
6746n/aImplement pickling support.
6747n/a[clinic start generated code]*/
6748n/a
6749n/astatic PyObject *
6750n/a_pickle_UnpicklerMemoProxy___reduce___impl(UnpicklerMemoProxyObject *self)
6751n/a/*[clinic end generated code: output=6da34ac048d94cca input=6920862413407199]*/
6752n/a{
6753n/a PyObject *reduce_value;
6754n/a PyObject *constructor_args;
6755n/a PyObject *contents = _pickle_UnpicklerMemoProxy_copy_impl(self);
6756n/a if (contents == NULL)
6757n/a return NULL;
6758n/a
6759n/a reduce_value = PyTuple_New(2);
6760n/a if (reduce_value == NULL) {
6761n/a Py_DECREF(contents);
6762n/a return NULL;
6763n/a }
6764n/a constructor_args = PyTuple_New(1);
6765n/a if (constructor_args == NULL) {
6766n/a Py_DECREF(contents);
6767n/a Py_DECREF(reduce_value);
6768n/a return NULL;
6769n/a }
6770n/a PyTuple_SET_ITEM(constructor_args, 0, contents);
6771n/a Py_INCREF((PyObject *)&PyDict_Type);
6772n/a PyTuple_SET_ITEM(reduce_value, 0, (PyObject *)&PyDict_Type);
6773n/a PyTuple_SET_ITEM(reduce_value, 1, constructor_args);
6774n/a return reduce_value;
6775n/a}
6776n/a
6777n/astatic PyMethodDef unpicklerproxy_methods[] = {
6778n/a _PICKLE_UNPICKLERMEMOPROXY_CLEAR_METHODDEF
6779n/a _PICKLE_UNPICKLERMEMOPROXY_COPY_METHODDEF
6780n/a _PICKLE_UNPICKLERMEMOPROXY___REDUCE___METHODDEF
6781n/a {NULL, NULL} /* sentinel */
6782n/a};
6783n/a
6784n/astatic void
6785n/aUnpicklerMemoProxy_dealloc(UnpicklerMemoProxyObject *self)
6786n/a{
6787n/a PyObject_GC_UnTrack(self);
6788n/a Py_XDECREF(self->unpickler);
6789n/a PyObject_GC_Del((PyObject *)self);
6790n/a}
6791n/a
6792n/astatic int
6793n/aUnpicklerMemoProxy_traverse(UnpicklerMemoProxyObject *self,
6794n/a visitproc visit, void *arg)
6795n/a{
6796n/a Py_VISIT(self->unpickler);
6797n/a return 0;
6798n/a}
6799n/a
6800n/astatic int
6801n/aUnpicklerMemoProxy_clear(UnpicklerMemoProxyObject *self)
6802n/a{
6803n/a Py_CLEAR(self->unpickler);
6804n/a return 0;
6805n/a}
6806n/a
6807n/astatic PyTypeObject UnpicklerMemoProxyType = {
6808n/a PyVarObject_HEAD_INIT(NULL, 0)
6809n/a "_pickle.UnpicklerMemoProxy", /*tp_name*/
6810n/a sizeof(UnpicklerMemoProxyObject), /*tp_basicsize*/
6811n/a 0,
6812n/a (destructor)UnpicklerMemoProxy_dealloc, /* tp_dealloc */
6813n/a 0, /* tp_print */
6814n/a 0, /* tp_getattr */
6815n/a 0, /* tp_setattr */
6816n/a 0, /* tp_compare */
6817n/a 0, /* tp_repr */
6818n/a 0, /* tp_as_number */
6819n/a 0, /* tp_as_sequence */
6820n/a 0, /* tp_as_mapping */
6821n/a PyObject_HashNotImplemented, /* tp_hash */
6822n/a 0, /* tp_call */
6823n/a 0, /* tp_str */
6824n/a PyObject_GenericGetAttr, /* tp_getattro */
6825n/a PyObject_GenericSetAttr, /* tp_setattro */
6826n/a 0, /* tp_as_buffer */
6827n/a Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
6828n/a 0, /* tp_doc */
6829n/a (traverseproc)UnpicklerMemoProxy_traverse, /* tp_traverse */
6830n/a (inquiry)UnpicklerMemoProxy_clear, /* tp_clear */
6831n/a 0, /* tp_richcompare */
6832n/a 0, /* tp_weaklistoffset */
6833n/a 0, /* tp_iter */
6834n/a 0, /* tp_iternext */
6835n/a unpicklerproxy_methods, /* tp_methods */
6836n/a};
6837n/a
6838n/astatic PyObject *
6839n/aUnpicklerMemoProxy_New(UnpicklerObject *unpickler)
6840n/a{
6841n/a UnpicklerMemoProxyObject *self;
6842n/a
6843n/a self = PyObject_GC_New(UnpicklerMemoProxyObject,
6844n/a &UnpicklerMemoProxyType);
6845n/a if (self == NULL)
6846n/a return NULL;
6847n/a Py_INCREF(unpickler);
6848n/a self->unpickler = unpickler;
6849n/a PyObject_GC_Track(self);
6850n/a return (PyObject *)self;
6851n/a}
6852n/a
6853n/a/*****************************************************************************/
6854n/a
6855n/a
6856n/astatic PyObject *
6857n/aUnpickler_get_memo(UnpicklerObject *self)
6858n/a{
6859n/a return UnpicklerMemoProxy_New(self);
6860n/a}
6861n/a
6862n/astatic int
6863n/aUnpickler_set_memo(UnpicklerObject *self, PyObject *obj)
6864n/a{
6865n/a PyObject **new_memo;
6866n/a Py_ssize_t new_memo_size = 0;
6867n/a Py_ssize_t i;
6868n/a
6869n/a if (obj == NULL) {
6870n/a PyErr_SetString(PyExc_TypeError,
6871n/a "attribute deletion is not supported");
6872n/a return -1;
6873n/a }
6874n/a
6875n/a if (Py_TYPE(obj) == &UnpicklerMemoProxyType) {
6876n/a UnpicklerObject *unpickler =
6877n/a ((UnpicklerMemoProxyObject *)obj)->unpickler;
6878n/a
6879n/a new_memo_size = unpickler->memo_size;
6880n/a new_memo = _Unpickler_NewMemo(new_memo_size);
6881n/a if (new_memo == NULL)
6882n/a return -1;
6883n/a
6884n/a for (i = 0; i < new_memo_size; i++) {
6885n/a Py_XINCREF(unpickler->memo[i]);
6886n/a new_memo[i] = unpickler->memo[i];
6887n/a }
6888n/a }
6889n/a else if (PyDict_Check(obj)) {
6890n/a Py_ssize_t i = 0;
6891n/a PyObject *key, *value;
6892n/a
6893n/a new_memo_size = PyDict_GET_SIZE(obj);
6894n/a new_memo = _Unpickler_NewMemo(new_memo_size);
6895n/a if (new_memo == NULL)
6896n/a return -1;
6897n/a
6898n/a while (PyDict_Next(obj, &i, &key, &value)) {
6899n/a Py_ssize_t idx;
6900n/a if (!PyLong_Check(key)) {
6901n/a PyErr_SetString(PyExc_TypeError,
6902n/a "memo key must be integers");
6903n/a goto error;
6904n/a }
6905n/a idx = PyLong_AsSsize_t(key);
6906n/a if (idx == -1 && PyErr_Occurred())
6907n/a goto error;
6908n/a if (idx < 0) {
6909n/a PyErr_SetString(PyExc_ValueError,
6910n/a "memo key must be positive integers.");
6911n/a goto error;
6912n/a }
6913n/a if (_Unpickler_MemoPut(self, idx, value) < 0)
6914n/a goto error;
6915n/a }
6916n/a }
6917n/a else {
6918n/a PyErr_Format(PyExc_TypeError,
6919n/a "'memo' attribute must be an UnpicklerMemoProxy object"
6920n/a "or dict, not %.200s", Py_TYPE(obj)->tp_name);
6921n/a return -1;
6922n/a }
6923n/a
6924n/a _Unpickler_MemoCleanup(self);
6925n/a self->memo_size = new_memo_size;
6926n/a self->memo = new_memo;
6927n/a
6928n/a return 0;
6929n/a
6930n/a error:
6931n/a if (new_memo_size) {
6932n/a i = new_memo_size;
6933n/a while (--i >= 0) {
6934n/a Py_XDECREF(new_memo[i]);
6935n/a }
6936n/a PyMem_FREE(new_memo);
6937n/a }
6938n/a return -1;
6939n/a}
6940n/a
6941n/astatic PyObject *
6942n/aUnpickler_get_persload(UnpicklerObject *self)
6943n/a{
6944n/a if (self->pers_func == NULL)
6945n/a PyErr_SetString(PyExc_AttributeError, "persistent_load");
6946n/a else
6947n/a Py_INCREF(self->pers_func);
6948n/a return self->pers_func;
6949n/a}
6950n/a
6951n/astatic int
6952n/aUnpickler_set_persload(UnpicklerObject *self, PyObject *value)
6953n/a{
6954n/a if (value == NULL) {
6955n/a PyErr_SetString(PyExc_TypeError,
6956n/a "attribute deletion is not supported");
6957n/a return -1;
6958n/a }
6959n/a if (!PyCallable_Check(value)) {
6960n/a PyErr_SetString(PyExc_TypeError,
6961n/a "persistent_load must be a callable taking "
6962n/a "one argument");
6963n/a return -1;
6964n/a }
6965n/a
6966n/a Py_INCREF(value);
6967n/a Py_XSETREF(self->pers_func, value);
6968n/a
6969n/a return 0;
6970n/a}
6971n/a
6972n/astatic PyGetSetDef Unpickler_getsets[] = {
6973n/a {"memo", (getter)Unpickler_get_memo, (setter)Unpickler_set_memo},
6974n/a {"persistent_load", (getter)Unpickler_get_persload,
6975n/a (setter)Unpickler_set_persload},
6976n/a {NULL}
6977n/a};
6978n/a
6979n/astatic PyTypeObject Unpickler_Type = {
6980n/a PyVarObject_HEAD_INIT(NULL, 0)
6981n/a "_pickle.Unpickler", /*tp_name*/
6982n/a sizeof(UnpicklerObject), /*tp_basicsize*/
6983n/a 0, /*tp_itemsize*/
6984n/a (destructor)Unpickler_dealloc, /*tp_dealloc*/
6985n/a 0, /*tp_print*/
6986n/a 0, /*tp_getattr*/
6987n/a 0, /*tp_setattr*/
6988n/a 0, /*tp_reserved*/
6989n/a 0, /*tp_repr*/
6990n/a 0, /*tp_as_number*/
6991n/a 0, /*tp_as_sequence*/
6992n/a 0, /*tp_as_mapping*/
6993n/a 0, /*tp_hash*/
6994n/a 0, /*tp_call*/
6995n/a 0, /*tp_str*/
6996n/a 0, /*tp_getattro*/
6997n/a 0, /*tp_setattro*/
6998n/a 0, /*tp_as_buffer*/
6999n/a Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
7000n/a _pickle_Unpickler___init____doc__, /*tp_doc*/
7001n/a (traverseproc)Unpickler_traverse, /*tp_traverse*/
7002n/a (inquiry)Unpickler_clear, /*tp_clear*/
7003n/a 0, /*tp_richcompare*/
7004n/a 0, /*tp_weaklistoffset*/
7005n/a 0, /*tp_iter*/
7006n/a 0, /*tp_iternext*/
7007n/a Unpickler_methods, /*tp_methods*/
7008n/a 0, /*tp_members*/
7009n/a Unpickler_getsets, /*tp_getset*/
7010n/a 0, /*tp_base*/
7011n/a 0, /*tp_dict*/
7012n/a 0, /*tp_descr_get*/
7013n/a 0, /*tp_descr_set*/
7014n/a 0, /*tp_dictoffset*/
7015n/a _pickle_Unpickler___init__, /*tp_init*/
7016n/a PyType_GenericAlloc, /*tp_alloc*/
7017n/a PyType_GenericNew, /*tp_new*/
7018n/a PyObject_GC_Del, /*tp_free*/
7019n/a 0, /*tp_is_gc*/
7020n/a};
7021n/a
7022n/a/*[clinic input]
7023n/a
7024n/a_pickle.dump
7025n/a
7026n/a obj: object
7027n/a file: object
7028n/a protocol: object = NULL
7029n/a *
7030n/a fix_imports: bool = True
7031n/a
7032n/aWrite a pickled representation of obj to the open file object file.
7033n/a
7034n/aThis is equivalent to ``Pickler(file, protocol).dump(obj)``, but may
7035n/abe more efficient.
7036n/a
7037n/aThe optional *protocol* argument tells the pickler to use the given
7038n/aprotocol supported protocols are 0, 1, 2, 3 and 4. The default
7039n/aprotocol is 3; a backward-incompatible protocol designed for Python 3.
7040n/a
7041n/aSpecifying a negative protocol version selects the highest protocol
7042n/aversion supported. The higher the protocol used, the more recent the
7043n/aversion of Python needed to read the pickle produced.
7044n/a
7045n/aThe *file* argument must have a write() method that accepts a single
7046n/abytes argument. It can thus be a file object opened for binary
7047n/awriting, an io.BytesIO instance, or any other custom object that meets
7048n/athis interface.
7049n/a
7050n/aIf *fix_imports* is True and protocol is less than 3, pickle will try
7051n/ato map the new Python 3 names to the old module names used in Python
7052n/a2, so that the pickle data stream is readable with Python 2.
7053n/a[clinic start generated code]*/
7054n/a
7055n/astatic PyObject *
7056n/a_pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file,
7057n/a PyObject *protocol, int fix_imports)
7058n/a/*[clinic end generated code: output=a4774d5fde7d34de input=830f8a64cef6f042]*/
7059n/a{
7060n/a PicklerObject *pickler = _Pickler_New();
7061n/a
7062n/a if (pickler == NULL)
7063n/a return NULL;
7064n/a
7065n/a if (_Pickler_SetProtocol(pickler, protocol, fix_imports) < 0)
7066n/a goto error;
7067n/a
7068n/a if (_Pickler_SetOutputStream(pickler, file) < 0)
7069n/a goto error;
7070n/a
7071n/a if (dump(pickler, obj) < 0)
7072n/a goto error;
7073n/a
7074n/a if (_Pickler_FlushToFile(pickler) < 0)
7075n/a goto error;
7076n/a
7077n/a Py_DECREF(pickler);
7078n/a Py_RETURN_NONE;
7079n/a
7080n/a error:
7081n/a Py_XDECREF(pickler);
7082n/a return NULL;
7083n/a}
7084n/a
7085n/a/*[clinic input]
7086n/a
7087n/a_pickle.dumps
7088n/a
7089n/a obj: object
7090n/a protocol: object = NULL
7091n/a *
7092n/a fix_imports: bool = True
7093n/a
7094n/aReturn the pickled representation of the object as a bytes object.
7095n/a
7096n/aThe optional *protocol* argument tells the pickler to use the given
7097n/aprotocol; supported protocols are 0, 1, 2, 3 and 4. The default
7098n/aprotocol is 3; a backward-incompatible protocol designed for Python 3.
7099n/a
7100n/aSpecifying a negative protocol version selects the highest protocol
7101n/aversion supported. The higher the protocol used, the more recent the
7102n/aversion of Python needed to read the pickle produced.
7103n/a
7104n/aIf *fix_imports* is True and *protocol* is less than 3, pickle will
7105n/atry to map the new Python 3 names to the old module names used in
7106n/aPython 2, so that the pickle data stream is readable with Python 2.
7107n/a[clinic start generated code]*/
7108n/a
7109n/astatic PyObject *
7110n/a_pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol,
7111n/a int fix_imports)
7112n/a/*[clinic end generated code: output=d75d5cda456fd261 input=293dbeda181580b7]*/
7113n/a{
7114n/a PyObject *result;
7115n/a PicklerObject *pickler = _Pickler_New();
7116n/a
7117n/a if (pickler == NULL)
7118n/a return NULL;
7119n/a
7120n/a if (_Pickler_SetProtocol(pickler, protocol, fix_imports) < 0)
7121n/a goto error;
7122n/a
7123n/a if (dump(pickler, obj) < 0)
7124n/a goto error;
7125n/a
7126n/a result = _Pickler_GetString(pickler);
7127n/a Py_DECREF(pickler);
7128n/a return result;
7129n/a
7130n/a error:
7131n/a Py_XDECREF(pickler);
7132n/a return NULL;
7133n/a}
7134n/a
7135n/a/*[clinic input]
7136n/a
7137n/a_pickle.load
7138n/a
7139n/a file: object
7140n/a *
7141n/a fix_imports: bool = True
7142n/a encoding: str = 'ASCII'
7143n/a errors: str = 'strict'
7144n/a
7145n/aRead and return an object from the pickle data stored in a file.
7146n/a
7147n/aThis is equivalent to ``Unpickler(file).load()``, but may be more
7148n/aefficient.
7149n/a
7150n/aThe protocol version of the pickle is detected automatically, so no
7151n/aprotocol argument is needed. Bytes past the pickled object's
7152n/arepresentation are ignored.
7153n/a
7154n/aThe argument *file* must have two methods, a read() method that takes
7155n/aan integer argument, and a readline() method that requires no
7156n/aarguments. Both methods should return bytes. Thus *file* can be a
7157n/abinary file object opened for reading, an io.BytesIO object, or any
7158n/aother custom object that meets this interface.
7159n/a
7160n/aOptional keyword arguments are *fix_imports*, *encoding* and *errors*,
7161n/awhich are used to control compatibility support for pickle stream
7162n/agenerated by Python 2. If *fix_imports* is True, pickle will try to
7163n/amap the old Python 2 names to the new names used in Python 3. The
7164n/a*encoding* and *errors* tell pickle how to decode 8-bit string
7165n/ainstances pickled by Python 2; these default to 'ASCII' and 'strict',
7166n/arespectively. The *encoding* can be 'bytes' to read these 8-bit
7167n/astring instances as bytes objects.
7168n/a[clinic start generated code]*/
7169n/a
7170n/astatic PyObject *
7171n/a_pickle_load_impl(PyObject *module, PyObject *file, int fix_imports,
7172n/a const char *encoding, const char *errors)
7173n/a/*[clinic end generated code: output=69e298160285199e input=01b44dd3fc07afa7]*/
7174n/a{
7175n/a PyObject *result;
7176n/a UnpicklerObject *unpickler = _Unpickler_New();
7177n/a
7178n/a if (unpickler == NULL)
7179n/a return NULL;
7180n/a
7181n/a if (_Unpickler_SetInputStream(unpickler, file) < 0)
7182n/a goto error;
7183n/a
7184n/a if (_Unpickler_SetInputEncoding(unpickler, encoding, errors) < 0)
7185n/a goto error;
7186n/a
7187n/a unpickler->fix_imports = fix_imports;
7188n/a
7189n/a result = load(unpickler);
7190n/a Py_DECREF(unpickler);
7191n/a return result;
7192n/a
7193n/a error:
7194n/a Py_XDECREF(unpickler);
7195n/a return NULL;
7196n/a}
7197n/a
7198n/a/*[clinic input]
7199n/a
7200n/a_pickle.loads
7201n/a
7202n/a data: object
7203n/a *
7204n/a fix_imports: bool = True
7205n/a encoding: str = 'ASCII'
7206n/a errors: str = 'strict'
7207n/a
7208n/aRead and return an object from the given pickle data.
7209n/a
7210n/aThe protocol version of the pickle is detected automatically, so no
7211n/aprotocol argument is needed. Bytes past the pickled object's
7212n/arepresentation are ignored.
7213n/a
7214n/aOptional keyword arguments are *fix_imports*, *encoding* and *errors*,
7215n/awhich are used to control compatibility support for pickle stream
7216n/agenerated by Python 2. If *fix_imports* is True, pickle will try to
7217n/amap the old Python 2 names to the new names used in Python 3. The
7218n/a*encoding* and *errors* tell pickle how to decode 8-bit string
7219n/ainstances pickled by Python 2; these default to 'ASCII' and 'strict',
7220n/arespectively. The *encoding* can be 'bytes' to read these 8-bit
7221n/astring instances as bytes objects.
7222n/a[clinic start generated code]*/
7223n/a
7224n/astatic PyObject *
7225n/a_pickle_loads_impl(PyObject *module, PyObject *data, int fix_imports,
7226n/a const char *encoding, const char *errors)
7227n/a/*[clinic end generated code: output=1e7cb2343f2c440f input=70605948a719feb9]*/
7228n/a{
7229n/a PyObject *result;
7230n/a UnpicklerObject *unpickler = _Unpickler_New();
7231n/a
7232n/a if (unpickler == NULL)
7233n/a return NULL;
7234n/a
7235n/a if (_Unpickler_SetStringInput(unpickler, data) < 0)
7236n/a goto error;
7237n/a
7238n/a if (_Unpickler_SetInputEncoding(unpickler, encoding, errors) < 0)
7239n/a goto error;
7240n/a
7241n/a unpickler->fix_imports = fix_imports;
7242n/a
7243n/a result = load(unpickler);
7244n/a Py_DECREF(unpickler);
7245n/a return result;
7246n/a
7247n/a error:
7248n/a Py_XDECREF(unpickler);
7249n/a return NULL;
7250n/a}
7251n/a
7252n/astatic struct PyMethodDef pickle_methods[] = {
7253n/a _PICKLE_DUMP_METHODDEF
7254n/a _PICKLE_DUMPS_METHODDEF
7255n/a _PICKLE_LOAD_METHODDEF
7256n/a _PICKLE_LOADS_METHODDEF
7257n/a {NULL, NULL} /* sentinel */
7258n/a};
7259n/a
7260n/astatic int
7261n/apickle_clear(PyObject *m)
7262n/a{
7263n/a _Pickle_ClearState(_Pickle_GetState(m));
7264n/a return 0;
7265n/a}
7266n/a
7267n/astatic void
7268n/apickle_free(PyObject *m)
7269n/a{
7270n/a _Pickle_ClearState(_Pickle_GetState(m));
7271n/a}
7272n/a
7273n/astatic int
7274n/apickle_traverse(PyObject *m, visitproc visit, void *arg)
7275n/a{
7276n/a PickleState *st = _Pickle_GetState(m);
7277n/a Py_VISIT(st->PickleError);
7278n/a Py_VISIT(st->PicklingError);
7279n/a Py_VISIT(st->UnpicklingError);
7280n/a Py_VISIT(st->dispatch_table);
7281n/a Py_VISIT(st->extension_registry);
7282n/a Py_VISIT(st->extension_cache);
7283n/a Py_VISIT(st->inverted_registry);
7284n/a Py_VISIT(st->name_mapping_2to3);
7285n/a Py_VISIT(st->import_mapping_2to3);
7286n/a Py_VISIT(st->name_mapping_3to2);
7287n/a Py_VISIT(st->import_mapping_3to2);
7288n/a Py_VISIT(st->codecs_encode);
7289n/a Py_VISIT(st->getattr);
7290n/a return 0;
7291n/a}
7292n/a
7293n/astatic struct PyModuleDef _picklemodule = {
7294n/a PyModuleDef_HEAD_INIT,
7295n/a "_pickle", /* m_name */
7296n/a pickle_module_doc, /* m_doc */
7297n/a sizeof(PickleState), /* m_size */
7298n/a pickle_methods, /* m_methods */
7299n/a NULL, /* m_reload */
7300n/a pickle_traverse, /* m_traverse */
7301n/a pickle_clear, /* m_clear */
7302n/a (freefunc)pickle_free /* m_free */
7303n/a};
7304n/a
7305n/aPyMODINIT_FUNC
7306n/aPyInit__pickle(void)
7307n/a{
7308n/a PyObject *m;
7309n/a PickleState *st;
7310n/a
7311n/a m = PyState_FindModule(&_picklemodule);
7312n/a if (m) {
7313n/a Py_INCREF(m);
7314n/a return m;
7315n/a }
7316n/a
7317n/a if (PyType_Ready(&Unpickler_Type) < 0)
7318n/a return NULL;
7319n/a if (PyType_Ready(&Pickler_Type) < 0)
7320n/a return NULL;
7321n/a if (PyType_Ready(&Pdata_Type) < 0)
7322n/a return NULL;
7323n/a if (PyType_Ready(&PicklerMemoProxyType) < 0)
7324n/a return NULL;
7325n/a if (PyType_Ready(&UnpicklerMemoProxyType) < 0)
7326n/a return NULL;
7327n/a
7328n/a /* Create the module and add the functions. */
7329n/a m = PyModule_Create(&_picklemodule);
7330n/a if (m == NULL)
7331n/a return NULL;
7332n/a
7333n/a Py_INCREF(&Pickler_Type);
7334n/a if (PyModule_AddObject(m, "Pickler", (PyObject *)&Pickler_Type) < 0)
7335n/a return NULL;
7336n/a Py_INCREF(&Unpickler_Type);
7337n/a if (PyModule_AddObject(m, "Unpickler", (PyObject *)&Unpickler_Type) < 0)
7338n/a return NULL;
7339n/a
7340n/a st = _Pickle_GetState(m);
7341n/a
7342n/a /* Initialize the exceptions. */
7343n/a st->PickleError = PyErr_NewException("_pickle.PickleError", NULL, NULL);
7344n/a if (st->PickleError == NULL)
7345n/a return NULL;
7346n/a st->PicklingError = \
7347n/a PyErr_NewException("_pickle.PicklingError", st->PickleError, NULL);
7348n/a if (st->PicklingError == NULL)
7349n/a return NULL;
7350n/a st->UnpicklingError = \
7351n/a PyErr_NewException("_pickle.UnpicklingError", st->PickleError, NULL);
7352n/a if (st->UnpicklingError == NULL)
7353n/a return NULL;
7354n/a
7355n/a Py_INCREF(st->PickleError);
7356n/a if (PyModule_AddObject(m, "PickleError", st->PickleError) < 0)
7357n/a return NULL;
7358n/a Py_INCREF(st->PicklingError);
7359n/a if (PyModule_AddObject(m, "PicklingError", st->PicklingError) < 0)
7360n/a return NULL;
7361n/a Py_INCREF(st->UnpicklingError);
7362n/a if (PyModule_AddObject(m, "UnpicklingError", st->UnpicklingError) < 0)
7363n/a return NULL;
7364n/a
7365n/a if (_Pickle_InitState(st) < 0)
7366n/a return NULL;
7367n/a
7368n/a return m;
7369n/a}