Python code coverage for Python/codecs.c

#	count	content
1	n/a	/* ------------------------------------------------------------------------
2	n/a
3	n/a	Python Codec Registry and support functions
4	n/a
5	n/a	Written by Marc-Andre Lemburg (mal@lemburg.com).
6	n/a
7	n/a	Copyright (c) Corporation for National Research Initiatives.
8	n/a
9	n/a	------------------------------------------------------------------------ */
10	n/a
11	n/a	#include "Python.h"
12	n/a	#include "ucnhash.h"
13	n/a	#include <ctype.h>
14	n/a
15	n/a	const char *Py_hexdigits = "0123456789abcdef";
16	n/a
17	n/a	/* --- Codec Registry ----------------------------------------------------- */
18	n/a
19	n/a	/* Import the standard encodings package which will register the first
20	n/a	codec search function.
21	n/a
22	n/a	This is done in a lazy way so that the Unicode implementation does
23	n/a	not downgrade startup time of scripts not needing it.
24	n/a
25	n/a	ImportErrors are silently ignored by this function. Only one try is
26	n/a	made.
27	n/a
28	n/a	*/
29	n/a
30	n/a	static int _PyCodecRegistry_Init(void); /* Forward */
31	n/a
32	n/a	int PyCodec_Register(PyObject *search_function)
33	n/a	{
34	n/a	PyInterpreterState *interp = PyThreadState_GET()->interp;
35	n/a	if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())
36	n/a	goto onError;
37	n/a	if (search_function == NULL) {
38	n/a	PyErr_BadArgument();
39	n/a	goto onError;
40	n/a	}
41	n/a	if (!PyCallable_Check(search_function)) {
42	n/a	PyErr_SetString(PyExc_TypeError, "argument must be callable");
43	n/a	goto onError;
44	n/a	}
45	n/a	return PyList_Append(interp->codec_search_path, search_function);
46	n/a
47	n/a	onError:
48	n/a	return -1;
49	n/a	}
50	n/a
51	n/a	/* Convert a string to a normalized Python string: all characters are
52	n/a	converted to lower case, spaces are replaced with underscores. */
53	n/a
54	n/a	static
55	n/a	PyObject normalizestring(const char string)
56	n/a	{
57	n/a	size_t i;
58	n/a	size_t len = strlen(string);
59	n/a	char *p;
60	n/a	PyObject *v;
61	n/a
62	n/a	if (len > PY_SSIZE_T_MAX) {
63	n/a	PyErr_SetString(PyExc_OverflowError, "string is too large");
64	n/a	return NULL;
65	n/a	}
66	n/a
67	n/a	p = PyMem_Malloc(len + 1);
68	n/a	if (p == NULL)
69	n/a	return PyErr_NoMemory();
70	n/a	for (i = 0; i < len; i++) {
71	n/a	char ch = string[i];
72	n/a	if (ch == ' ')
73	n/a	ch = '-';
74	n/a	else
75	n/a	ch = Py_TOLOWER(Py_CHARMASK(ch));
76	n/a	p[i] = ch;
77	n/a	}
78	n/a	p[i] = '\0';
79	n/a	v = PyUnicode_FromString(p);
80	n/a	if (v == NULL)
81	n/a	return NULL;
82	n/a	PyMem_Free(p);
83	n/a	return v;
84	n/a	}
85	n/a
86	n/a	/* Lookup the given encoding and return a tuple providing the codec
87	n/a	facilities.
88	n/a
89	n/a	The encoding string is looked up converted to all lower-case
90	n/a	characters. This makes encodings looked up through this mechanism
91	n/a	effectively case-insensitive.
92	n/a
93	n/a	If no codec is found, a LookupError is set and NULL returned.
94	n/a
95	n/a	As side effect, this tries to load the encodings package, if not
96	n/a	yet done. This is part of the lazy load strategy for the encodings
97	n/a	package.
98	n/a
99	n/a	*/
100	n/a
101	n/a	PyObject _PyCodec_Lookup(const char encoding)
102	n/a	{
103	n/a	PyInterpreterState *interp;
104	n/a	PyObject result, args = NULL, *v;
105	n/a	Py_ssize_t i, len;
106	n/a
107	n/a	if (encoding == NULL) {
108	n/a	PyErr_BadArgument();
109	n/a	goto onError;
110	n/a	}
111	n/a
112	n/a	interp = PyThreadState_GET()->interp;
113	n/a	if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())
114	n/a	goto onError;
115	n/a
116	n/a	/* Convert the encoding to a normalized Python string: all
117	n/a	characters are converted to lower case, spaces and hyphens are
118	n/a	replaced with underscores. */
119	n/a	v = normalizestring(encoding);
120	n/a	if (v == NULL)
121	n/a	goto onError;
122	n/a	PyUnicode_InternInPlace(&v);
123	n/a
124	n/a	/* First, try to lookup the name in the registry dictionary */
125	n/a	result = PyDict_GetItem(interp->codec_search_cache, v);
126	n/a	if (result != NULL) {
127	n/a	Py_INCREF(result);
128	n/a	Py_DECREF(v);
129	n/a	return result;
130	n/a	}
131	n/a
132	n/a	/* Next, scan the search functions in order of registration */
133	n/a	args = PyTuple_New(1);
134	n/a	if (args == NULL)
135	n/a	goto onError;
136	n/a	PyTuple_SET_ITEM(args,0,v);
137	n/a
138	n/a	len = PyList_Size(interp->codec_search_path);
139	n/a	if (len < 0)
140	n/a	goto onError;
141	n/a	if (len == 0) {
142	n/a	PyErr_SetString(PyExc_LookupError,
143	n/a	"no codec search functions registered: "
144	n/a	"can't find encoding");
145	n/a	goto onError;
146	n/a	}
147	n/a
148	n/a	for (i = 0; i < len; i++) {
149	n/a	PyObject *func;
150	n/a
151	n/a	func = PyList_GetItem(interp->codec_search_path, i);
152	n/a	if (func == NULL)
153	n/a	goto onError;
154	n/a	result = PyEval_CallObject(func, args);
155	n/a	if (result == NULL)
156	n/a	goto onError;
157	n/a	if (result == Py_None) {
158	n/a	Py_DECREF(result);
159	n/a	continue;
160	n/a	}
161	n/a	if (!PyTuple_Check(result) \|\| PyTuple_GET_SIZE(result) != 4) {
162	n/a	PyErr_SetString(PyExc_TypeError,
163	n/a	"codec search functions must return 4-tuples");
164	n/a	Py_DECREF(result);
165	n/a	goto onError;
166	n/a	}
167	n/a	break;
168	n/a	}
169	n/a	if (i == len) {
170	n/a	/* XXX Perhaps we should cache misses too ? */
171	n/a	PyErr_Format(PyExc_LookupError,
172	n/a	"unknown encoding: %s", encoding);
173	n/a	goto onError;
174	n/a	}
175	n/a
176	n/a	/* Cache and return the result */
177	n/a	if (PyDict_SetItem(interp->codec_search_cache, v, result) < 0) {
178	n/a	Py_DECREF(result);
179	n/a	goto onError;
180	n/a	}
181	n/a	Py_DECREF(args);
182	n/a	return result;
183	n/a
184	n/a	onError:
185	n/a	Py_XDECREF(args);
186	n/a	return NULL;
187	n/a	}
188	n/a
189	n/a	int _PyCodec_Forget(const char *encoding)
190	n/a	{
191	n/a	PyInterpreterState *interp;
192	n/a	PyObject *v;
193	n/a	int result;
194	n/a
195	n/a	interp = PyThreadState_GET()->interp;
196	n/a	if (interp->codec_search_path == NULL) {
197	n/a	return -1;
198	n/a	}
199	n/a
200	n/a	/* Convert the encoding to a normalized Python string: all
201	n/a	characters are converted to lower case, spaces and hyphens are
202	n/a	replaced with underscores. */
203	n/a	v = normalizestring(encoding);
204	n/a	if (v == NULL) {
205	n/a	return -1;
206	n/a	}
207	n/a
208	n/a	/* Drop the named codec from the internal cache */
209	n/a	result = PyDict_DelItem(interp->codec_search_cache, v);
210	n/a	Py_DECREF(v);
211	n/a
212	n/a	return result;
213	n/a	}
214	n/a
215	n/a	/* Codec registry encoding check API. */
216	n/a
217	n/a	int PyCodec_KnownEncoding(const char *encoding)
218	n/a	{
219	n/a	PyObject *codecs;
220	n/a
221	n/a	codecs = _PyCodec_Lookup(encoding);
222	n/a	if (!codecs) {
223	n/a	PyErr_Clear();
224	n/a	return 0;
225	n/a	}
226	n/a	else {
227	n/a	Py_DECREF(codecs);
228	n/a	return 1;
229	n/a	}
230	n/a	}
231	n/a
232	n/a	static
233	n/a	PyObject args_tuple(PyObject object,
234	n/a	const char *errors)
235	n/a	{
236	n/a	PyObject *args;
237	n/a
238	n/a	args = PyTuple_New(1 + (errors != NULL));
239	n/a	if (args == NULL)
240	n/a	return NULL;
241	n/a	Py_INCREF(object);
242	n/a	PyTuple_SET_ITEM(args,0,object);
243	n/a	if (errors) {
244	n/a	PyObject *v;
245	n/a
246	n/a	v = PyUnicode_FromString(errors);
247	n/a	if (v == NULL) {
248	n/a	Py_DECREF(args);
249	n/a	return NULL;
250	n/a	}
251	n/a	PyTuple_SET_ITEM(args, 1, v);
252	n/a	}
253	n/a	return args;
254	n/a	}
255	n/a
256	n/a	/* Helper function to get a codec item */
257	n/a
258	n/a	static
259	n/a	PyObject codec_getitem(const char encoding, int index)
260	n/a	{
261	n/a	PyObject *codecs;
262	n/a	PyObject *v;
263	n/a
264	n/a	codecs = _PyCodec_Lookup(encoding);
265	n/a	if (codecs == NULL)
266	n/a	return NULL;
267	n/a	v = PyTuple_GET_ITEM(codecs, index);
268	n/a	Py_DECREF(codecs);
269	n/a	Py_INCREF(v);
270	n/a	return v;
271	n/a	}
272	n/a
273	n/a	/* Helper functions to create an incremental codec. */
274	n/a	static
275	n/a	PyObject codec_makeincrementalcodec(PyObject codec_info,
276	n/a	const char *errors,
277	n/a	const char *attrname)
278	n/a	{
279	n/a	PyObject ret, inccodec;
280	n/a
281	n/a	inccodec = PyObject_GetAttrString(codec_info, attrname);
282	n/a	if (inccodec == NULL)
283	n/a	return NULL;
284	n/a	if (errors)
285	n/a	ret = PyObject_CallFunction(inccodec, "s", errors);
286	n/a	else
287	n/a	ret = _PyObject_CallNoArg(inccodec);
288	n/a	Py_DECREF(inccodec);
289	n/a	return ret;
290	n/a	}
291	n/a
292	n/a	static
293	n/a	PyObject codec_getincrementalcodec(const char encoding,
294	n/a	const char *errors,
295	n/a	const char *attrname)
296	n/a	{
297	n/a	PyObject codec_info, ret;
298	n/a
299	n/a	codec_info = _PyCodec_Lookup(encoding);
300	n/a	if (codec_info == NULL)
301	n/a	return NULL;
302	n/a	ret = codec_makeincrementalcodec(codec_info, errors, attrname);
303	n/a	Py_DECREF(codec_info);
304	n/a	return ret;
305	n/a	}
306	n/a
307	n/a	/* Helper function to create a stream codec. */
308	n/a
309	n/a	static
310	n/a	PyObject codec_getstreamcodec(const char encoding,
311	n/a	PyObject *stream,
312	n/a	const char *errors,
313	n/a	const int index)
314	n/a	{
315	n/a	PyObject codecs, streamcodec, *codeccls;
316	n/a
317	n/a	codecs = _PyCodec_Lookup(encoding);
318	n/a	if (codecs == NULL)
319	n/a	return NULL;
320	n/a
321	n/a	codeccls = PyTuple_GET_ITEM(codecs, index);
322	n/a	if (errors != NULL)
323	n/a	streamcodec = PyObject_CallFunction(codeccls, "Os", stream, errors);
324	n/a	else
325	n/a	streamcodec = PyObject_CallFunctionObjArgs(codeccls, stream, NULL);
326	n/a	Py_DECREF(codecs);
327	n/a	return streamcodec;
328	n/a	}
329	n/a
330	n/a	/* Helpers to work with the result of _PyCodec_Lookup
331	n/a
332	n/a	*/
333	n/a	PyObject _PyCodecInfo_GetIncrementalDecoder(PyObject codec_info,
334	n/a	const char *errors)
335	n/a	{
336	n/a	return codec_makeincrementalcodec(codec_info, errors,
337	n/a	"incrementaldecoder");
338	n/a	}
339	n/a
340	n/a	PyObject _PyCodecInfo_GetIncrementalEncoder(PyObject codec_info,
341	n/a	const char *errors)
342	n/a	{
343	n/a	return codec_makeincrementalcodec(codec_info, errors,
344	n/a	"incrementalencoder");
345	n/a	}
346	n/a
347	n/a
348	n/a	/* Convenience APIs to query the Codec registry.
349	n/a
350	n/a	All APIs return a codec object with incremented refcount.
351	n/a
352	n/a	*/
353	n/a
354	n/a	PyObject PyCodec_Encoder(const char encoding)
355	n/a	{
356	n/a	return codec_getitem(encoding, 0);
357	n/a	}
358	n/a
359	n/a	PyObject PyCodec_Decoder(const char encoding)
360	n/a	{
361	n/a	return codec_getitem(encoding, 1);
362	n/a	}
363	n/a
364	n/a	PyObject PyCodec_IncrementalEncoder(const char encoding,
365	n/a	const char *errors)
366	n/a	{
367	n/a	return codec_getincrementalcodec(encoding, errors, "incrementalencoder");
368	n/a	}
369	n/a
370	n/a	PyObject PyCodec_IncrementalDecoder(const char encoding,
371	n/a	const char *errors)
372	n/a	{
373	n/a	return codec_getincrementalcodec(encoding, errors, "incrementaldecoder");
374	n/a	}
375	n/a
376	n/a	PyObject PyCodec_StreamReader(const char encoding,
377	n/a	PyObject *stream,
378	n/a	const char *errors)
379	n/a	{
380	n/a	return codec_getstreamcodec(encoding, stream, errors, 2);
381	n/a	}
382	n/a
383	n/a	PyObject PyCodec_StreamWriter(const char encoding,
384	n/a	PyObject *stream,
385	n/a	const char *errors)
386	n/a	{
387	n/a	return codec_getstreamcodec(encoding, stream, errors, 3);
388	n/a	}
389	n/a
390	n/a	/* Helper that tries to ensure the reported exception chain indicates the
391	n/a	* codec that was invoked to trigger the failure without changing the type
392	n/a	* of the exception raised.
393	n/a	*/
394	n/a	static void
395	n/a	wrap_codec_error(const char *operation,
396	n/a	const char *encoding)
397	n/a	{
398	n/a	/* TrySetFromCause will replace the active exception with a suitably
399	n/a	* updated clone if it can, otherwise it will leave the original
400	n/a	* exception alone.
401	n/a	*/
402	n/a	_PyErr_TrySetFromCause("%s with '%s' codec failed",
403	n/a	operation, encoding);
404	n/a	}
405	n/a
406	n/a	/* Encode an object (e.g. a Unicode object) using the given encoding
407	n/a	and return the resulting encoded object (usually a Python string).
408	n/a
409	n/a	errors is passed to the encoder factory as argument if non-NULL. */
410	n/a
411	n/a	static PyObject *
412	n/a	_PyCodec_EncodeInternal(PyObject *object,
413	n/a	PyObject *encoder,
414	n/a	const char *encoding,
415	n/a	const char *errors)
416	n/a	{
417	n/a	PyObject args = NULL, result = NULL;
418	n/a	PyObject *v = NULL;
419	n/a
420	n/a	args = args_tuple(object, errors);
421	n/a	if (args == NULL)
422	n/a	goto onError;
423	n/a
424	n/a	result = PyEval_CallObject(encoder, args);
425	n/a	if (result == NULL) {
426	n/a	wrap_codec_error("encoding", encoding);
427	n/a	goto onError;
428	n/a	}
429	n/a
430	n/a	if (!PyTuple_Check(result) \|\|
431	n/a	PyTuple_GET_SIZE(result) != 2) {
432	n/a	PyErr_SetString(PyExc_TypeError,
433	n/a	"encoder must return a tuple (object, integer)");
434	n/a	goto onError;
435	n/a	}
436	n/a	v = PyTuple_GET_ITEM(result,0);
437	n/a	Py_INCREF(v);
438	n/a	/* We don't check or use the second (integer) entry. */
439	n/a
440	n/a	Py_DECREF(args);
441	n/a	Py_DECREF(encoder);
442	n/a	Py_DECREF(result);
443	n/a	return v;
444	n/a
445	n/a	onError:
446	n/a	Py_XDECREF(result);
447	n/a	Py_XDECREF(args);
448	n/a	Py_XDECREF(encoder);
449	n/a	return NULL;
450	n/a	}
451	n/a
452	n/a	/* Decode an object (usually a Python string) using the given encoding
453	n/a	and return an equivalent object (e.g. a Unicode object).
454	n/a
455	n/a	errors is passed to the decoder factory as argument if non-NULL. */
456	n/a
457	n/a	static PyObject *
458	n/a	_PyCodec_DecodeInternal(PyObject *object,
459	n/a	PyObject *decoder,
460	n/a	const char *encoding,
461	n/a	const char *errors)
462	n/a	{
463	n/a	PyObject args = NULL, result = NULL;
464	n/a	PyObject *v;
465	n/a
466	n/a	args = args_tuple(object, errors);
467	n/a	if (args == NULL)
468	n/a	goto onError;
469	n/a
470	n/a	result = PyEval_CallObject(decoder,args);
471	n/a	if (result == NULL) {
472	n/a	wrap_codec_error("decoding", encoding);
473	n/a	goto onError;
474	n/a	}
475	n/a	if (!PyTuple_Check(result) \|\|
476	n/a	PyTuple_GET_SIZE(result) != 2) {
477	n/a	PyErr_SetString(PyExc_TypeError,
478	n/a	"decoder must return a tuple (object,integer)");
479	n/a	goto onError;
480	n/a	}
481	n/a	v = PyTuple_GET_ITEM(result,0);
482	n/a	Py_INCREF(v);
483	n/a	/* We don't check or use the second (integer) entry. */
484	n/a
485	n/a	Py_DECREF(args);
486	n/a	Py_DECREF(decoder);
487	n/a	Py_DECREF(result);
488	n/a	return v;
489	n/a
490	n/a	onError:
491	n/a	Py_XDECREF(args);
492	n/a	Py_XDECREF(decoder);
493	n/a	Py_XDECREF(result);
494	n/a	return NULL;
495	n/a	}
496	n/a
497	n/a	/* Generic encoding/decoding API */
498	n/a	PyObject PyCodec_Encode(PyObject object,
499	n/a	const char *encoding,
500	n/a	const char *errors)
501	n/a	{
502	n/a	PyObject *encoder;
503	n/a
504	n/a	encoder = PyCodec_Encoder(encoding);
505	n/a	if (encoder == NULL)
506	n/a	return NULL;
507	n/a
508	n/a	return _PyCodec_EncodeInternal(object, encoder, encoding, errors);
509	n/a	}
510	n/a
511	n/a	PyObject PyCodec_Decode(PyObject object,
512	n/a	const char *encoding,
513	n/a	const char *errors)
514	n/a	{
515	n/a	PyObject *decoder;
516	n/a
517	n/a	decoder = PyCodec_Decoder(encoding);
518	n/a	if (decoder == NULL)
519	n/a	return NULL;
520	n/a
521	n/a	return _PyCodec_DecodeInternal(object, decoder, encoding, errors);
522	n/a	}
523	n/a
524	n/a	/* Text encoding/decoding API */
525	n/a	PyObject * _PyCodec_LookupTextEncoding(const char *encoding,
526	n/a	const char *alternate_command)
527	n/a	{
528	n/a	_Py_IDENTIFIER(_is_text_encoding);
529	n/a	PyObject *codec;
530	n/a	PyObject *attr;
531	n/a	int is_text_codec;
532	n/a
533	n/a	codec = _PyCodec_Lookup(encoding);
534	n/a	if (codec == NULL)
535	n/a	return NULL;
536	n/a
537	n/a	/* Backwards compatibility: assume any raw tuple describes a text
538	n/a	* encoding, and the same for anything lacking the private
539	n/a	* attribute.
540	n/a	*/
541	n/a	if (!PyTuple_CheckExact(codec)) {
542	n/a	attr = _PyObject_GetAttrId(codec, &PyId__is_text_encoding);
543	n/a	if (attr == NULL) {
544	n/a	if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
545	n/a	PyErr_Clear();
546	n/a	} else {
547	n/a	Py_DECREF(codec);
548	n/a	return NULL;
549	n/a	}
550	n/a	} else {
551	n/a	is_text_codec = PyObject_IsTrue(attr);
552	n/a	Py_DECREF(attr);
553	n/a	if (is_text_codec <= 0) {
554	n/a	Py_DECREF(codec);
555	n/a	if (!is_text_codec)
556	n/a	PyErr_Format(PyExc_LookupError,
557	n/a	"'%.400s' is not a text encoding; "
558	n/a	"use %s to handle arbitrary codecs",
559	n/a	encoding, alternate_command);
560	n/a	return NULL;
561	n/a	}
562	n/a	}
563	n/a	}
564	n/a
565	n/a	/* This appears to be a valid text encoding */
566	n/a	return codec;
567	n/a	}
568	n/a
569	n/a
570	n/a	static
571	n/a	PyObject codec_getitem_checked(const char encoding,
572	n/a	const char *alternate_command,
573	n/a	int index)
574	n/a	{
575	n/a	PyObject *codec;
576	n/a	PyObject *v;
577	n/a
578	n/a	codec = _PyCodec_LookupTextEncoding(encoding, alternate_command);
579	n/a	if (codec == NULL)
580	n/a	return NULL;
581	n/a
582	n/a	v = PyTuple_GET_ITEM(codec, index);
583	n/a	Py_INCREF(v);
584	n/a	Py_DECREF(codec);
585	n/a	return v;
586	n/a	}
587	n/a
588	n/a	static PyObject * _PyCodec_TextEncoder(const char *encoding)
589	n/a	{
590	n/a	return codec_getitem_checked(encoding, "codecs.encode()", 0);
591	n/a	}
592	n/a
593	n/a	static PyObject * _PyCodec_TextDecoder(const char *encoding)
594	n/a	{
595	n/a	return codec_getitem_checked(encoding, "codecs.decode()", 1);
596	n/a	}
597	n/a
598	n/a	PyObject _PyCodec_EncodeText(PyObject object,
599	n/a	const char *encoding,
600	n/a	const char *errors)
601	n/a	{
602	n/a	PyObject *encoder;
603	n/a
604	n/a	encoder = _PyCodec_TextEncoder(encoding);
605	n/a	if (encoder == NULL)
606	n/a	return NULL;
607	n/a
608	n/a	return _PyCodec_EncodeInternal(object, encoder, encoding, errors);
609	n/a	}
610	n/a
611	n/a	PyObject _PyCodec_DecodeText(PyObject object,
612	n/a	const char *encoding,
613	n/a	const char *errors)
614	n/a	{
615	n/a	PyObject *decoder;
616	n/a
617	n/a	decoder = _PyCodec_TextDecoder(encoding);
618	n/a	if (decoder == NULL)
619	n/a	return NULL;
620	n/a
621	n/a	return _PyCodec_DecodeInternal(object, decoder, encoding, errors);
622	n/a	}
623	n/a
624	n/a	/* Register the error handling callback function error under the name
625	n/a	name. This function will be called by the codec when it encounters
626	n/a	an unencodable characters/undecodable bytes and doesn't know the
627	n/a	callback name, when name is specified as the error parameter
628	n/a	in the call to the encode/decode function.
629	n/a	Return 0 on success, -1 on error */
630	n/a	int PyCodec_RegisterError(const char name, PyObject error)
631	n/a	{
632	n/a	PyInterpreterState *interp = PyThreadState_GET()->interp;
633	n/a	if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())
634	n/a	return -1;
635	n/a	if (!PyCallable_Check(error)) {
636	n/a	PyErr_SetString(PyExc_TypeError, "handler must be callable");
637	n/a	return -1;
638	n/a	}
639	n/a	return PyDict_SetItemString(interp->codec_error_registry,
640	n/a	name, error);
641	n/a	}
642	n/a
643	n/a	/* Lookup the error handling callback function registered under the
644	n/a	name error. As a special case NULL can be passed, in which case
645	n/a	the error handling callback for strict encoding will be returned. */
646	n/a	PyObject PyCodec_LookupError(const char name)
647	n/a	{
648	n/a	PyObject *handler = NULL;
649	n/a
650	n/a	PyInterpreterState *interp = PyThreadState_GET()->interp;
651	n/a	if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())
652	n/a	return NULL;
653	n/a
654	n/a	if (name==NULL)
655	n/a	name = "strict";
656	n/a	handler = PyDict_GetItemString(interp->codec_error_registry, name);
657	n/a	if (!handler)
658	n/a	PyErr_Format(PyExc_LookupError, "unknown error handler name '%.400s'", name);
659	n/a	else
660	n/a	Py_INCREF(handler);
661	n/a	return handler;
662	n/a	}
663	n/a
664	n/a	static void wrong_exception_type(PyObject *exc)
665	n/a	{
666	n/a	PyErr_Format(PyExc_TypeError,
667	n/a	"don't know how to handle %.200s in error callback",
668	n/a	exc->ob_type->tp_name);
669	n/a	}
670	n/a
671	n/a	PyObject PyCodec_StrictErrors(PyObject exc)
672	n/a	{
673	n/a	if (PyExceptionInstance_Check(exc))
674	n/a	PyErr_SetObject(PyExceptionInstance_Class(exc), exc);
675	n/a	else
676	n/a	PyErr_SetString(PyExc_TypeError, "codec must pass exception instance");
677	n/a	return NULL;
678	n/a	}
679	n/a
680	n/a
681	n/a	PyObject PyCodec_IgnoreErrors(PyObject exc)
682	n/a	{
683	n/a	Py_ssize_t end;
684	n/a
685	n/a	if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) {
686	n/a	if (PyUnicodeEncodeError_GetEnd(exc, &end))
687	n/a	return NULL;
688	n/a	}
689	n/a	else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) {
690	n/a	if (PyUnicodeDecodeError_GetEnd(exc, &end))
691	n/a	return NULL;
692	n/a	}
693	n/a	else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError)) {
694	n/a	if (PyUnicodeTranslateError_GetEnd(exc, &end))
695	n/a	return NULL;
696	n/a	}
697	n/a	else {
698	n/a	wrong_exception_type(exc);
699	n/a	return NULL;
700	n/a	}
701	n/a	return Py_BuildValue("(Nn)", PyUnicode_New(0, 0), end);
702	n/a	}
703	n/a
704	n/a
705	n/a	PyObject PyCodec_ReplaceErrors(PyObject exc)
706	n/a	{
707	n/a	Py_ssize_t start, end, i, len;
708	n/a
709	n/a	if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) {
710	n/a	PyObject *res;
711	n/a	int kind;
712	n/a	void *data;
713	n/a	if (PyUnicodeEncodeError_GetStart(exc, &start))
714	n/a	return NULL;
715	n/a	if (PyUnicodeEncodeError_GetEnd(exc, &end))
716	n/a	return NULL;
717	n/a	len = end - start;
718	n/a	res = PyUnicode_New(len, '?');
719	n/a	if (res == NULL)
720	n/a	return NULL;
721	n/a	kind = PyUnicode_KIND(res);
722	n/a	data = PyUnicode_DATA(res);
723	n/a	for (i = 0; i < len; ++i)
724	n/a	PyUnicode_WRITE(kind, data, i, '?');
725	n/a	assert(_PyUnicode_CheckConsistency(res, 1));
726	n/a	return Py_BuildValue("(Nn)", res, end);
727	n/a	}
728	n/a	else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) {
729	n/a	if (PyUnicodeDecodeError_GetEnd(exc, &end))
730	n/a	return NULL;
731	n/a	return Py_BuildValue("(Cn)",
732	n/a	(int)Py_UNICODE_REPLACEMENT_CHARACTER,
733	n/a	end);
734	n/a	}
735	n/a	else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError)) {
736	n/a	PyObject *res;
737	n/a	int kind;
738	n/a	void *data;
739	n/a	if (PyUnicodeTranslateError_GetStart(exc, &start))
740	n/a	return NULL;
741	n/a	if (PyUnicodeTranslateError_GetEnd(exc, &end))
742	n/a	return NULL;
743	n/a	len = end - start;
744	n/a	res = PyUnicode_New(len, Py_UNICODE_REPLACEMENT_CHARACTER);
745	n/a	if (res == NULL)
746	n/a	return NULL;
747	n/a	kind = PyUnicode_KIND(res);
748	n/a	data = PyUnicode_DATA(res);
749	n/a	for (i=0; i < len; i++)
750	n/a	PyUnicode_WRITE(kind, data, i, Py_UNICODE_REPLACEMENT_CHARACTER);
751	n/a	assert(_PyUnicode_CheckConsistency(res, 1));
752	n/a	return Py_BuildValue("(Nn)", res, end);
753	n/a	}
754	n/a	else {
755	n/a	wrong_exception_type(exc);
756	n/a	return NULL;
757	n/a	}
758	n/a	}
759	n/a
760	n/a	PyObject PyCodec_XMLCharRefReplaceErrors(PyObject exc)
761	n/a	{
762	n/a	if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) {
763	n/a	PyObject *restuple;
764	n/a	PyObject *object;
765	n/a	Py_ssize_t i;
766	n/a	Py_ssize_t start;
767	n/a	Py_ssize_t end;
768	n/a	PyObject *res;
769	n/a	unsigned char *outp;
770	n/a	Py_ssize_t ressize;
771	n/a	Py_UCS4 ch;
772	n/a	if (PyUnicodeEncodeError_GetStart(exc, &start))
773	n/a	return NULL;
774	n/a	if (PyUnicodeEncodeError_GetEnd(exc, &end))
775	n/a	return NULL;
776	n/a	if (!(object = PyUnicodeEncodeError_GetObject(exc)))
777	n/a	return NULL;
778	n/a	if (end - start > PY_SSIZE_T_MAX / (2+7+1))
779	n/a	end = start + PY_SSIZE_T_MAX / (2+7+1);
780	n/a	for (i = start, ressize = 0; i < end; ++i) {
781	n/a	/* object is guaranteed to be "ready" */
782	n/a	ch = PyUnicode_READ_CHAR(object, i);
783	n/a	if (ch<10)
784	n/a	ressize += 2+1+1;
785	n/a	else if (ch<100)
786	n/a	ressize += 2+2+1;
787	n/a	else if (ch<1000)
788	n/a	ressize += 2+3+1;
789	n/a	else if (ch<10000)
790	n/a	ressize += 2+4+1;
791	n/a	else if (ch<100000)
792	n/a	ressize += 2+5+1;
793	n/a	else if (ch<1000000)
794	n/a	ressize += 2+6+1;
795	n/a	else
796	n/a	ressize += 2+7+1;
797	n/a	}
798	n/a	/* allocate replacement */
799	n/a	res = PyUnicode_New(ressize, 127);
800	n/a	if (res == NULL) {
801	n/a	Py_DECREF(object);
802	n/a	return NULL;
803	n/a	}
804	n/a	outp = PyUnicode_1BYTE_DATA(res);
805	n/a	/* generate replacement */
806	n/a	for (i = start; i < end; ++i) {
807	n/a	int digits;
808	n/a	int base;
809	n/a	ch = PyUnicode_READ_CHAR(object, i);
810	n/a	*outp++ = '&';
811	n/a	*outp++ = '#';
812	n/a	if (ch<10) {
813	n/a	digits = 1;
814	n/a	base = 1;
815	n/a	}
816	n/a	else if (ch<100) {
817	n/a	digits = 2;
818	n/a	base = 10;
819	n/a	}
820	n/a	else if (ch<1000) {
821	n/a	digits = 3;
822	n/a	base = 100;
823	n/a	}
824	n/a	else if (ch<10000) {
825	n/a	digits = 4;
826	n/a	base = 1000;
827	n/a	}
828	n/a	else if (ch<100000) {
829	n/a	digits = 5;
830	n/a	base = 10000;
831	n/a	}
832	n/a	else if (ch<1000000) {
833	n/a	digits = 6;
834	n/a	base = 100000;
835	n/a	}
836	n/a	else {
837	n/a	digits = 7;
838	n/a	base = 1000000;
839	n/a	}
840	n/a	while (digits-->0) {
841	n/a	*outp++ = '0' + ch/base;
842	n/a	ch %= base;
843	n/a	base /= 10;
844	n/a	}
845	n/a	*outp++ = ';';
846	n/a	}
847	n/a	assert(_PyUnicode_CheckConsistency(res, 1));
848	n/a	restuple = Py_BuildValue("(Nn)", res, end);
849	n/a	Py_DECREF(object);
850	n/a	return restuple;
851	n/a	}
852	n/a	else {
853	n/a	wrong_exception_type(exc);
854	n/a	return NULL;
855	n/a	}
856	n/a	}
857	n/a
858	n/a	PyObject PyCodec_BackslashReplaceErrors(PyObject exc)
859	n/a	{
860	n/a	PyObject *object;
861	n/a	Py_ssize_t i;
862	n/a	Py_ssize_t start;
863	n/a	Py_ssize_t end;
864	n/a	PyObject *res;
865	n/a	unsigned char *outp;
866	n/a	int ressize;
867	n/a	Py_UCS4 c;
868	n/a
869	n/a	if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) {
870	n/a	const unsigned char *p;
871	n/a	if (PyUnicodeDecodeError_GetStart(exc, &start))
872	n/a	return NULL;
873	n/a	if (PyUnicodeDecodeError_GetEnd(exc, &end))
874	n/a	return NULL;
875	n/a	if (!(object = PyUnicodeDecodeError_GetObject(exc)))
876	n/a	return NULL;
877	n/a	p = (const unsigned char*)PyBytes_AS_STRING(object);
878	n/a	res = PyUnicode_New(4 * (end - start), 127);
879	n/a	if (res == NULL) {
880	n/a	Py_DECREF(object);
881	n/a	return NULL;
882	n/a	}
883	n/a	outp = PyUnicode_1BYTE_DATA(res);
884	n/a	for (i = start; i < end; i++, outp += 4) {
885	n/a	unsigned char c = p[i];
886	n/a	outp[0] = '\\';
887	n/a	outp[1] = 'x';
888	n/a	outp[2] = Py_hexdigits[(c>>4)&0xf];
889	n/a	outp[3] = Py_hexdigits[c&0xf];
890	n/a	}
891	n/a
892	n/a	assert(_PyUnicode_CheckConsistency(res, 1));
893	n/a	Py_DECREF(object);
894	n/a	return Py_BuildValue("(Nn)", res, end);
895	n/a	}
896	n/a	if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) {
897	n/a	if (PyUnicodeEncodeError_GetStart(exc, &start))
898	n/a	return NULL;
899	n/a	if (PyUnicodeEncodeError_GetEnd(exc, &end))
900	n/a	return NULL;
901	n/a	if (!(object = PyUnicodeEncodeError_GetObject(exc)))
902	n/a	return NULL;
903	n/a	}
904	n/a	else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError)) {
905	n/a	if (PyUnicodeTranslateError_GetStart(exc, &start))
906	n/a	return NULL;
907	n/a	if (PyUnicodeTranslateError_GetEnd(exc, &end))
908	n/a	return NULL;
909	n/a	if (!(object = PyUnicodeTranslateError_GetObject(exc)))
910	n/a	return NULL;
911	n/a	}
912	n/a	else {
913	n/a	wrong_exception_type(exc);
914	n/a	return NULL;
915	n/a	}
916	n/a
917	n/a	if (end - start > PY_SSIZE_T_MAX / (1+1+8))
918	n/a	end = start + PY_SSIZE_T_MAX / (1+1+8);
919	n/a	for (i = start, ressize = 0; i < end; ++i) {
920	n/a	/* object is guaranteed to be "ready" */
921	n/a	c = PyUnicode_READ_CHAR(object, i);
922	n/a	if (c >= 0x10000) {
923	n/a	ressize += 1+1+8;
924	n/a	}
925	n/a	else if (c >= 0x100) {
926	n/a	ressize += 1+1+4;
927	n/a	}
928	n/a	else
929	n/a	ressize += 1+1+2;
930	n/a	}
931	n/a	res = PyUnicode_New(ressize, 127);
932	n/a	if (res == NULL) {
933	n/a	Py_DECREF(object);
934	n/a	return NULL;
935	n/a	}
936	n/a	outp = PyUnicode_1BYTE_DATA(res);
937	n/a	for (i = start; i < end; ++i) {
938	n/a	c = PyUnicode_READ_CHAR(object, i);
939	n/a	*outp++ = '\\';
940	n/a	if (c >= 0x00010000) {
941	n/a	*outp++ = 'U';
942	n/a	*outp++ = Py_hexdigits[(c>>28)&0xf];
943	n/a	*outp++ = Py_hexdigits[(c>>24)&0xf];
944	n/a	*outp++ = Py_hexdigits[(c>>20)&0xf];
945	n/a	*outp++ = Py_hexdigits[(c>>16)&0xf];
946	n/a	*outp++ = Py_hexdigits[(c>>12)&0xf];
947	n/a	*outp++ = Py_hexdigits[(c>>8)&0xf];
948	n/a	}
949	n/a	else if (c >= 0x100) {
950	n/a	*outp++ = 'u';
951	n/a	*outp++ = Py_hexdigits[(c>>12)&0xf];
952	n/a	*outp++ = Py_hexdigits[(c>>8)&0xf];
953	n/a	}
954	n/a	else
955	n/a	*outp++ = 'x';
956	n/a	*outp++ = Py_hexdigits[(c>>4)&0xf];
957	n/a	*outp++ = Py_hexdigits[c&0xf];
958	n/a	}
959	n/a
960	n/a	assert(_PyUnicode_CheckConsistency(res, 1));
961	n/a	Py_DECREF(object);
962	n/a	return Py_BuildValue("(Nn)", res, end);
963	n/a	}
964	n/a
965	n/a	static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL;
966	n/a
967	n/a	PyObject PyCodec_NameReplaceErrors(PyObject exc)
968	n/a	{
969	n/a	if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) {
970	n/a	PyObject *restuple;
971	n/a	PyObject *object;
972	n/a	Py_ssize_t i;
973	n/a	Py_ssize_t start;
974	n/a	Py_ssize_t end;
975	n/a	PyObject *res;
976	n/a	unsigned char *outp;
977	n/a	Py_ssize_t ressize;
978	n/a	int replsize;
979	n/a	Py_UCS4 c;
980	n/a	char buffer[256]; /* NAME_MAXLEN */
981	n/a	if (PyUnicodeEncodeError_GetStart(exc, &start))
982	n/a	return NULL;
983	n/a	if (PyUnicodeEncodeError_GetEnd(exc, &end))
984	n/a	return NULL;
985	n/a	if (!(object = PyUnicodeEncodeError_GetObject(exc)))
986	n/a	return NULL;
987	n/a	if (!ucnhash_CAPI) {
988	n/a	/* load the unicode data module */
989	n/a	ucnhash_CAPI = (_PyUnicode_Name_CAPI *)PyCapsule_Import(
990	n/a	PyUnicodeData_CAPSULE_NAME, 1);
991	n/a	if (!ucnhash_CAPI)
992	n/a	return NULL;
993	n/a	}
994	n/a	for (i = start, ressize = 0; i < end; ++i) {
995	n/a	/* object is guaranteed to be "ready" */
996	n/a	c = PyUnicode_READ_CHAR(object, i);
997	n/a	if (ucnhash_CAPI->getname(NULL, c, buffer, sizeof(buffer), 1)) {
998	n/a	replsize = 1+1+1+(int)strlen(buffer)+1;
999	n/a	}
1000	n/a	else if (c >= 0x10000) {
1001	n/a	replsize = 1+1+8;
1002	n/a	}
1003	n/a	else if (c >= 0x100) {
1004	n/a	replsize = 1+1+4;
1005	n/a	}
1006	n/a	else
1007	n/a	replsize = 1+1+2;
1008	n/a	if (ressize > PY_SSIZE_T_MAX - replsize)
1009	n/a	break;
1010	n/a	ressize += replsize;
1011	n/a	}
1012	n/a	end = i;
1013	n/a	res = PyUnicode_New(ressize, 127);
1014	n/a	if (res==NULL)
1015	n/a	return NULL;
1016	n/a	for (i = start, outp = PyUnicode_1BYTE_DATA(res);
1017	n/a	i < end; ++i) {
1018	n/a	c = PyUnicode_READ_CHAR(object, i);
1019	n/a	*outp++ = '\\';
1020	n/a	if (ucnhash_CAPI->getname(NULL, c, buffer, sizeof(buffer), 1)) {
1021	n/a	*outp++ = 'N';
1022	n/a	*outp++ = '{';
1023	n/a	strcpy((char *)outp, buffer);
1024	n/a	outp += strlen(buffer);
1025	n/a	*outp++ = '}';
1026	n/a	continue;
1027	n/a	}
1028	n/a	if (c >= 0x00010000) {
1029	n/a	*outp++ = 'U';
1030	n/a	*outp++ = Py_hexdigits[(c>>28)&0xf];
1031	n/a	*outp++ = Py_hexdigits[(c>>24)&0xf];
1032	n/a	*outp++ = Py_hexdigits[(c>>20)&0xf];
1033	n/a	*outp++ = Py_hexdigits[(c>>16)&0xf];
1034	n/a	*outp++ = Py_hexdigits[(c>>12)&0xf];
1035	n/a	*outp++ = Py_hexdigits[(c>>8)&0xf];
1036	n/a	}
1037	n/a	else if (c >= 0x100) {
1038	n/a	*outp++ = 'u';
1039	n/a	*outp++ = Py_hexdigits[(c>>12)&0xf];
1040	n/a	*outp++ = Py_hexdigits[(c>>8)&0xf];
1041	n/a	}
1042	n/a	else
1043	n/a	*outp++ = 'x';
1044	n/a	*outp++ = Py_hexdigits[(c>>4)&0xf];
1045	n/a	*outp++ = Py_hexdigits[c&0xf];
1046	n/a	}
1047	n/a
1048	n/a	assert(outp == PyUnicode_1BYTE_DATA(res) + ressize);
1049	n/a	assert(_PyUnicode_CheckConsistency(res, 1));
1050	n/a	restuple = Py_BuildValue("(Nn)", res, end);
1051	n/a	Py_DECREF(object);
1052	n/a	return restuple;
1053	n/a	}
1054	n/a	else {
1055	n/a	wrong_exception_type(exc);
1056	n/a	return NULL;
1057	n/a	}
1058	n/a	}
1059	n/a
1060	n/a	#define ENC_UNKNOWN -1
1061	n/a	#define ENC_UTF8 0
1062	n/a	#define ENC_UTF16BE 1
1063	n/a	#define ENC_UTF16LE 2
1064	n/a	#define ENC_UTF32BE 3
1065	n/a	#define ENC_UTF32LE 4
1066	n/a
1067	n/a	static int
1068	n/a	get_standard_encoding(const char encoding, int bytelength)
1069	n/a	{
1070	n/a	if (Py_TOLOWER(encoding[0]) == 'u' &&
1071	n/a	Py_TOLOWER(encoding[1]) == 't' &&
1072	n/a	Py_TOLOWER(encoding[2]) == 'f') {
1073	n/a	encoding += 3;
1074	n/a	if (encoding == '-' \|\| encoding == '_' )
1075	n/a	encoding++;
1076	n/a	if (encoding[0] == '8' && encoding[1] == '\0') {
1077	n/a	*bytelength = 3;
1078	n/a	return ENC_UTF8;
1079	n/a	}
1080	n/a	else if (encoding[0] == '1' && encoding[1] == '6') {
1081	n/a	encoding += 2;
1082	n/a	*bytelength = 2;
1083	n/a	if (*encoding == '\0') {
1084	n/a	#ifdef WORDS_BIGENDIAN
1085	n/a	return ENC_UTF16BE;
1086	n/a	#else
1087	n/a	return ENC_UTF16LE;
1088	n/a	#endif
1089	n/a	}
1090	n/a	if (encoding == '-' \|\| encoding == '_' )
1091	n/a	encoding++;
1092	n/a	if (Py_TOLOWER(encoding[1]) == 'e' && encoding[2] == '\0') {
1093	n/a	if (Py_TOLOWER(encoding[0]) == 'b')
1094	n/a	return ENC_UTF16BE;
1095	n/a	if (Py_TOLOWER(encoding[0]) == 'l')
1096	n/a	return ENC_UTF16LE;
1097	n/a	}
1098	n/a	}
1099	n/a	else if (encoding[0] == '3' && encoding[1] == '2') {
1100	n/a	encoding += 2;
1101	n/a	*bytelength = 4;
1102	n/a	if (*encoding == '\0') {
1103	n/a	#ifdef WORDS_BIGENDIAN
1104	n/a	return ENC_UTF32BE;
1105	n/a	#else
1106	n/a	return ENC_UTF32LE;
1107	n/a	#endif
1108	n/a	}
1109	n/a	if (encoding == '-' \|\| encoding == '_' )
1110	n/a	encoding++;
1111	n/a	if (Py_TOLOWER(encoding[1]) == 'e' && encoding[2] == '\0') {
1112	n/a	if (Py_TOLOWER(encoding[0]) == 'b')
1113	n/a	return ENC_UTF32BE;
1114	n/a	if (Py_TOLOWER(encoding[0]) == 'l')
1115	n/a	return ENC_UTF32LE;
1116	n/a	}
1117	n/a	}
1118	n/a	}
1119	n/a	else if (strcmp(encoding, "CP_UTF8") == 0) {
1120	n/a	*bytelength = 3;
1121	n/a	return ENC_UTF8;
1122	n/a	}
1123	n/a	return ENC_UNKNOWN;
1124	n/a	}
1125	n/a
1126	n/a	/* This handler is declared static until someone demonstrates
1127	n/a	a need to call it directly. */
1128	n/a	static PyObject *
1129	n/a	PyCodec_SurrogatePassErrors(PyObject *exc)
1130	n/a	{
1131	n/a	PyObject *restuple;
1132	n/a	PyObject *object;
1133	n/a	PyObject *encode;
1134	n/a	const char *encoding;
1135	n/a	int code;
1136	n/a	int bytelength;
1137	n/a	Py_ssize_t i;
1138	n/a	Py_ssize_t start;
1139	n/a	Py_ssize_t end;
1140	n/a	PyObject *res;
1141	n/a
1142	n/a	if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) {
1143	n/a	unsigned char *outp;
1144	n/a	if (PyUnicodeEncodeError_GetStart(exc, &start))
1145	n/a	return NULL;
1146	n/a	if (PyUnicodeEncodeError_GetEnd(exc, &end))
1147	n/a	return NULL;
1148	n/a	if (!(object = PyUnicodeEncodeError_GetObject(exc)))
1149	n/a	return NULL;
1150	n/a	if (!(encode = PyUnicodeEncodeError_GetEncoding(exc))) {
1151	n/a	Py_DECREF(object);
1152	n/a	return NULL;
1153	n/a	}
1154	n/a	if (!(encoding = PyUnicode_AsUTF8(encode))) {
1155	n/a	Py_DECREF(object);
1156	n/a	Py_DECREF(encode);
1157	n/a	return NULL;
1158	n/a	}
1159	n/a	code = get_standard_encoding(encoding, &bytelength);
1160	n/a	Py_DECREF(encode);
1161	n/a	if (code == ENC_UNKNOWN) {
1162	n/a	/* Not supported, fail with original exception */
1163	n/a	PyErr_SetObject(PyExceptionInstance_Class(exc), exc);
1164	n/a	Py_DECREF(object);
1165	n/a	return NULL;
1166	n/a	}
1167	n/a
1168	n/a	if (end - start > PY_SSIZE_T_MAX / bytelength)
1169	n/a	end = start + PY_SSIZE_T_MAX / bytelength;
1170	n/a	res = PyBytes_FromStringAndSize(NULL, bytelength*(end-start));
1171	n/a	if (!res) {
1172	n/a	Py_DECREF(object);
1173	n/a	return NULL;
1174	n/a	}
1175	n/a	outp = (unsigned char*)PyBytes_AsString(res);
1176	n/a	for (i = start; i < end; i++) {
1177	n/a	/* object is guaranteed to be "ready" */
1178	n/a	Py_UCS4 ch = PyUnicode_READ_CHAR(object, i);
1179	n/a	if (!Py_UNICODE_IS_SURROGATE(ch)) {
1180	n/a	/* Not a surrogate, fail with original exception */
1181	n/a	PyErr_SetObject(PyExceptionInstance_Class(exc), exc);
1182	n/a	Py_DECREF(res);
1183	n/a	Py_DECREF(object);
1184	n/a	return NULL;
1185	n/a	}
1186	n/a	switch (code) {
1187	n/a	case ENC_UTF8:
1188	n/a	*outp++ = (unsigned char)(0xe0 \| (ch >> 12));
1189	n/a	*outp++ = (unsigned char)(0x80 \| ((ch >> 6) & 0x3f));
1190	n/a	*outp++ = (unsigned char)(0x80 \| (ch & 0x3f));
1191	n/a	break;
1192	n/a	case ENC_UTF16LE:
1193	n/a	*outp++ = (unsigned char) ch;
1194	n/a	*outp++ = (unsigned char)(ch >> 8);
1195	n/a	break;
1196	n/a	case ENC_UTF16BE:
1197	n/a	*outp++ = (unsigned char)(ch >> 8);
1198	n/a	*outp++ = (unsigned char) ch;
1199	n/a	break;
1200	n/a	case ENC_UTF32LE:
1201	n/a	*outp++ = (unsigned char) ch;
1202	n/a	*outp++ = (unsigned char)(ch >> 8);
1203	n/a	*outp++ = (unsigned char)(ch >> 16);
1204	n/a	*outp++ = (unsigned char)(ch >> 24);
1205	n/a	break;
1206	n/a	case ENC_UTF32BE:
1207	n/a	*outp++ = (unsigned char)(ch >> 24);
1208	n/a	*outp++ = (unsigned char)(ch >> 16);
1209	n/a	*outp++ = (unsigned char)(ch >> 8);
1210	n/a	*outp++ = (unsigned char) ch;
1211	n/a	break;
1212	n/a	}
1213	n/a	}
1214	n/a	restuple = Py_BuildValue("(On)", res, end);
1215	n/a	Py_DECREF(res);
1216	n/a	Py_DECREF(object);
1217	n/a	return restuple;
1218	n/a	}
1219	n/a	else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) {
1220	n/a	const unsigned char *p;
1221	n/a	Py_UCS4 ch = 0;
1222	n/a	if (PyUnicodeDecodeError_GetStart(exc, &start))
1223	n/a	return NULL;
1224	n/a	if (PyUnicodeDecodeError_GetEnd(exc, &end))
1225	n/a	return NULL;
1226	n/a	if (!(object = PyUnicodeDecodeError_GetObject(exc)))
1227	n/a	return NULL;
1228	n/a	p = (const unsigned char*)PyBytes_AS_STRING(object);
1229	n/a	if (!(encode = PyUnicodeDecodeError_GetEncoding(exc))) {
1230	n/a	Py_DECREF(object);
1231	n/a	return NULL;
1232	n/a	}
1233	n/a	if (!(encoding = PyUnicode_AsUTF8(encode))) {
1234	n/a	Py_DECREF(object);
1235	n/a	Py_DECREF(encode);
1236	n/a	return NULL;
1237	n/a	}
1238	n/a	code = get_standard_encoding(encoding, &bytelength);
1239	n/a	Py_DECREF(encode);
1240	n/a	if (code == ENC_UNKNOWN) {
1241	n/a	/* Not supported, fail with original exception */
1242	n/a	PyErr_SetObject(PyExceptionInstance_Class(exc), exc);
1243	n/a	Py_DECREF(object);
1244	n/a	return NULL;
1245	n/a	}
1246	n/a
1247	n/a	/* Try decoding a single surrogate character. If
1248	n/a	there are more, let the codec call us again. */
1249	n/a	p += start;
1250	n/a	if (PyBytes_GET_SIZE(object) - start >= bytelength) {
1251	n/a	switch (code) {
1252	n/a	case ENC_UTF8:
1253	n/a	if ((p[0] & 0xf0) == 0xe0 &&
1254	n/a	(p[1] & 0xc0) == 0x80 &&
1255	n/a	(p[2] & 0xc0) == 0x80) {
1256	n/a	/* it's a three-byte code */
1257	n/a	ch = ((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) + (p[2] & 0x3f);
1258	n/a	}
1259	n/a	break;
1260	n/a	case ENC_UTF16LE:
1261	n/a	ch = p[1] << 8 \| p[0];
1262	n/a	break;
1263	n/a	case ENC_UTF16BE:
1264	n/a	ch = p[0] << 8 \| p[1];
1265	n/a	break;
1266	n/a	case ENC_UTF32LE:
1267	n/a	ch = (p[3] << 24) \| (p[2] << 16) \| (p[1] << 8) \| p[0];
1268	n/a	break;
1269	n/a	case ENC_UTF32BE:
1270	n/a	ch = (p[0] << 24) \| (p[1] << 16) \| (p[2] << 8) \| p[3];
1271	n/a	break;
1272	n/a	}
1273	n/a	}
1274	n/a
1275	n/a	Py_DECREF(object);
1276	n/a	if (!Py_UNICODE_IS_SURROGATE(ch)) {
1277	n/a	/* it's not a surrogate - fail */
1278	n/a	PyErr_SetObject(PyExceptionInstance_Class(exc), exc);
1279	n/a	return NULL;
1280	n/a	}
1281	n/a	res = PyUnicode_FromOrdinal(ch);
1282	n/a	if (res == NULL)
1283	n/a	return NULL;
1284	n/a	return Py_BuildValue("(Nn)", res, start + bytelength);
1285	n/a	}
1286	n/a	else {
1287	n/a	wrong_exception_type(exc);
1288	n/a	return NULL;
1289	n/a	}
1290	n/a	}
1291	n/a
1292	n/a	static PyObject *
1293	n/a	PyCodec_SurrogateEscapeErrors(PyObject *exc)
1294	n/a	{
1295	n/a	PyObject *restuple;
1296	n/a	PyObject *object;
1297	n/a	Py_ssize_t i;
1298	n/a	Py_ssize_t start;
1299	n/a	Py_ssize_t end;
1300	n/a	PyObject *res;
1301	n/a
1302	n/a	if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) {
1303	n/a	char *outp;
1304	n/a	if (PyUnicodeEncodeError_GetStart(exc, &start))
1305	n/a	return NULL;
1306	n/a	if (PyUnicodeEncodeError_GetEnd(exc, &end))
1307	n/a	return NULL;
1308	n/a	if (!(object = PyUnicodeEncodeError_GetObject(exc)))
1309	n/a	return NULL;
1310	n/a	res = PyBytes_FromStringAndSize(NULL, end-start);
1311	n/a	if (!res) {
1312	n/a	Py_DECREF(object);
1313	n/a	return NULL;
1314	n/a	}
1315	n/a	outp = PyBytes_AsString(res);
1316	n/a	for (i = start; i < end; i++) {
1317	n/a	/* object is guaranteed to be "ready" */
1318	n/a	Py_UCS4 ch = PyUnicode_READ_CHAR(object, i);
1319	n/a	if (ch < 0xdc80 \|\| ch > 0xdcff) {
1320	n/a	/* Not a UTF-8b surrogate, fail with original exception */
1321	n/a	PyErr_SetObject(PyExceptionInstance_Class(exc), exc);
1322	n/a	Py_DECREF(res);
1323	n/a	Py_DECREF(object);
1324	n/a	return NULL;
1325	n/a	}
1326	n/a	*outp++ = ch - 0xdc00;
1327	n/a	}
1328	n/a	restuple = Py_BuildValue("(On)", res, end);
1329	n/a	Py_DECREF(res);
1330	n/a	Py_DECREF(object);
1331	n/a	return restuple;
1332	n/a	}
1333	n/a	else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) {
1334	n/a	PyObject *str;
1335	n/a	const unsigned char *p;
1336	n/a	Py_UCS2 ch[4]; /* decode up to 4 bad bytes. */
1337	n/a	int consumed = 0;
1338	n/a	if (PyUnicodeDecodeError_GetStart(exc, &start))
1339	n/a	return NULL;
1340	n/a	if (PyUnicodeDecodeError_GetEnd(exc, &end))
1341	n/a	return NULL;
1342	n/a	if (!(object = PyUnicodeDecodeError_GetObject(exc)))
1343	n/a	return NULL;
1344	n/a	p = (const unsigned char*)PyBytes_AS_STRING(object);
1345	n/a	while (consumed < 4 && consumed < end-start) {
1346	n/a	/* Refuse to escape ASCII bytes. */
1347	n/a	if (p[start+consumed] < 128)
1348	n/a	break;
1349	n/a	ch[consumed] = 0xdc00 + p[start+consumed];
1350	n/a	consumed++;
1351	n/a	}
1352	n/a	Py_DECREF(object);
1353	n/a	if (!consumed) {
1354	n/a	/* codec complained about ASCII byte. */
1355	n/a	PyErr_SetObject(PyExceptionInstance_Class(exc), exc);
1356	n/a	return NULL;
1357	n/a	}
1358	n/a	str = PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, ch, consumed);
1359	n/a	if (str == NULL)
1360	n/a	return NULL;
1361	n/a	return Py_BuildValue("(Nn)", str, start+consumed);
1362	n/a	}
1363	n/a	else {
1364	n/a	wrong_exception_type(exc);
1365	n/a	return NULL;
1366	n/a	}
1367	n/a	}
1368	n/a
1369	n/a
1370	n/a	static PyObject strict_errors(PyObject self, PyObject *exc)
1371	n/a	{
1372	n/a	return PyCodec_StrictErrors(exc);
1373	n/a	}
1374	n/a
1375	n/a
1376	n/a	static PyObject ignore_errors(PyObject self, PyObject *exc)
1377	n/a	{
1378	n/a	return PyCodec_IgnoreErrors(exc);
1379	n/a	}
1380	n/a
1381	n/a
1382	n/a	static PyObject replace_errors(PyObject self, PyObject *exc)
1383	n/a	{
1384	n/a	return PyCodec_ReplaceErrors(exc);
1385	n/a	}
1386	n/a
1387	n/a
1388	n/a	static PyObject xmlcharrefreplace_errors(PyObject self, PyObject *exc)
1389	n/a	{
1390	n/a	return PyCodec_XMLCharRefReplaceErrors(exc);
1391	n/a	}
1392	n/a
1393	n/a
1394	n/a	static PyObject backslashreplace_errors(PyObject self, PyObject *exc)
1395	n/a	{
1396	n/a	return PyCodec_BackslashReplaceErrors(exc);
1397	n/a	}
1398	n/a
1399	n/a	static PyObject namereplace_errors(PyObject self, PyObject *exc)
1400	n/a	{
1401	n/a	return PyCodec_NameReplaceErrors(exc);
1402	n/a	}
1403	n/a
1404	n/a	static PyObject surrogatepass_errors(PyObject self, PyObject *exc)
1405	n/a	{
1406	n/a	return PyCodec_SurrogatePassErrors(exc);
1407	n/a	}
1408	n/a
1409	n/a	static PyObject surrogateescape_errors(PyObject self, PyObject *exc)
1410	n/a	{
1411	n/a	return PyCodec_SurrogateEscapeErrors(exc);
1412	n/a	}
1413	n/a
1414	n/a	static int _PyCodecRegistry_Init(void)
1415	n/a	{
1416	n/a	static struct {
1417	n/a	char *name;
1418	n/a	PyMethodDef def;
1419	n/a	} methods[] =
1420	n/a	{
1421	n/a	{
1422	n/a	"strict",
1423	n/a	{
1424	n/a	"strict_errors",
1425	n/a	strict_errors,
1426	n/a	METH_O,
1427	n/a	PyDoc_STR("Implements the 'strict' error handling, which "
1428	n/a	"raises a UnicodeError on coding errors.")
1429	n/a	}
1430	n/a	},
1431	n/a	{
1432	n/a	"ignore",
1433	n/a	{
1434	n/a	"ignore_errors",
1435	n/a	ignore_errors,
1436	n/a	METH_O,
1437	n/a	PyDoc_STR("Implements the 'ignore' error handling, which "
1438	n/a	"ignores malformed data and continues.")
1439	n/a	}
1440	n/a	},
1441	n/a	{
1442	n/a	"replace",
1443	n/a	{
1444	n/a	"replace_errors",
1445	n/a	replace_errors,
1446	n/a	METH_O,
1447	n/a	PyDoc_STR("Implements the 'replace' error handling, which "
1448	n/a	"replaces malformed data with a replacement marker.")
1449	n/a	}
1450	n/a	},
1451	n/a	{
1452	n/a	"xmlcharrefreplace",
1453	n/a	{
1454	n/a	"xmlcharrefreplace_errors",
1455	n/a	xmlcharrefreplace_errors,
1456	n/a	METH_O,
1457	n/a	PyDoc_STR("Implements the 'xmlcharrefreplace' error handling, "
1458	n/a	"which replaces an unencodable character with the "
1459	n/a	"appropriate XML character reference.")
1460	n/a	}
1461	n/a	},
1462	n/a	{
1463	n/a	"backslashreplace",
1464	n/a	{
1465	n/a	"backslashreplace_errors",
1466	n/a	backslashreplace_errors,
1467	n/a	METH_O,
1468	n/a	PyDoc_STR("Implements the 'backslashreplace' error handling, "
1469	n/a	"which replaces malformed data with a backslashed "
1470	n/a	"escape sequence.")
1471	n/a	}
1472	n/a	},
1473	n/a	{
1474	n/a	"namereplace",
1475	n/a	{
1476	n/a	"namereplace_errors",
1477	n/a	namereplace_errors,
1478	n/a	METH_O,
1479	n/a	PyDoc_STR("Implements the 'namereplace' error handling, "
1480	n/a	"which replaces an unencodable character with a "
1481	n/a	"\\N{...} escape sequence.")
1482	n/a	}
1483	n/a	},
1484	n/a	{
1485	n/a	"surrogatepass",
1486	n/a	{
1487	n/a	"surrogatepass",
1488	n/a	surrogatepass_errors,
1489	n/a	METH_O
1490	n/a	}
1491	n/a	},
1492	n/a	{
1493	n/a	"surrogateescape",
1494	n/a	{
1495	n/a	"surrogateescape",
1496	n/a	surrogateescape_errors,
1497	n/a	METH_O
1498	n/a	}
1499	n/a	}
1500	n/a	};
1501	n/a
1502	n/a	PyInterpreterState *interp = PyThreadState_GET()->interp;
1503	n/a	PyObject *mod;
1504	n/a	unsigned i;
1505	n/a
1506	n/a	if (interp->codec_search_path != NULL)
1507	n/a	return 0;
1508	n/a
1509	n/a	interp->codec_search_path = PyList_New(0);
1510	n/a	interp->codec_search_cache = PyDict_New();
1511	n/a	interp->codec_error_registry = PyDict_New();
1512	n/a
1513	n/a	if (interp->codec_error_registry) {
1514	n/a	for (i = 0; i < Py_ARRAY_LENGTH(methods); ++i) {
1515	n/a	PyObject *func = PyCFunction_NewEx(&methods[i].def, NULL, NULL);
1516	n/a	int res;
1517	n/a	if (!func)
1518	n/a	Py_FatalError("can't initialize codec error registry");
1519	n/a	res = PyCodec_RegisterError(methods[i].name, func);
1520	n/a	Py_DECREF(func);
1521	n/a	if (res)
1522	n/a	Py_FatalError("can't initialize codec error registry");
1523	n/a	}
1524	n/a	}
1525	n/a
1526	n/a	if (interp->codec_search_path == NULL \|\|
1527	n/a	interp->codec_search_cache == NULL \|\|
1528	n/a	interp->codec_error_registry == NULL)
1529	n/a	Py_FatalError("can't initialize codec registry");
1530	n/a
1531	n/a	mod = PyImport_ImportModuleNoBlock("encodings");
1532	n/a	if (mod == NULL) {
1533	n/a	return -1;
1534	n/a	}
1535	n/a	Py_DECREF(mod);
1536	n/a	interp->codecs_initialized = 1;
1537	n/a	return 0;
1538	n/a	}