Python code coverage for Objects/unicodeobject.c

#	count	content
1	n/a	/*
2	n/a
3	n/a	Unicode implementation based on original code by Fredrik Lundh,
4	n/a	modified by Marc-Andre Lemburg <mal@lemburg.com>.
5	n/a
6	n/a	Major speed upgrades to the method implementations at the Reykjavik
7	n/a	NeedForSpeed sprint, by Fredrik Lundh and Andrew Dalke.
8	n/a
9	n/a	Copyright (c) Corporation for National Research Initiatives.
10	n/a
11	n/a	--------------------------------------------------------------------
12	n/a	The original string type implementation is:
13	n/a
14	n/a	Copyright (c) 1999 by Secret Labs AB
15	n/a	Copyright (c) 1999 by Fredrik Lundh
16	n/a
17	n/a	By obtaining, using, and/or copying this software and/or its
18	n/a	associated documentation, you agree that you have read, understood,
19	n/a	and will comply with the following terms and conditions:
20	n/a
21	n/a	Permission to use, copy, modify, and distribute this software and its
22	n/a	associated documentation for any purpose and without fee is hereby
23	n/a	granted, provided that the above copyright notice appears in all
24	n/a	copies, and that both that copyright notice and this permission notice
25	n/a	appear in supporting documentation, and that the name of Secret Labs
26	n/a	AB or the author not be used in advertising or publicity pertaining to
27	n/a	distribution of the software without specific, written prior
28	n/a	permission.
29	n/a
30	n/a	SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
31	n/a	THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
32	n/a	FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR
33	n/a	ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
34	n/a	WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
35	n/a	ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
36	n/a	OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
37	n/a	--------------------------------------------------------------------
38	n/a
39	n/a	*/
40	n/a
41	n/a	#define PY_SSIZE_T_CLEAN
42	n/a	#include "Python.h"
43	n/a	#include "ucnhash.h"
44	n/a	#include "bytes_methods.h"
45	n/a	#include "stringlib/eq.h"
46	n/a
47	n/a	#ifdef MS_WINDOWS
48	n/a	#include <windows.h>
49	n/a	#endif
50	n/a
51	n/a	/*[clinic input]
52	n/a	class str "PyObject *" "&PyUnicode_Type"
53	n/a	[clinic start generated code]*/
54	n/a	/[clinic end generated code: output=da39a3ee5e6b4b0d input=4884c934de622cf6]/
55	n/a
56	n/a	/*[python input]
57	n/a	class Py_UCS4_converter(CConverter):
58	n/a	type = 'Py_UCS4'
59	n/a	converter = 'convert_uc'
60	n/a
61	n/a	def converter_init(self):
62	n/a	if self.default is not unspecified:
63	n/a	self.c_default = ascii(self.default)
64	n/a	if len(self.c_default) > 4 or self.c_default[0] != "'":
65	n/a	self.c_default = hex(ord(self.default))
66	n/a
67	n/a	[python start generated code]*/
68	n/a	/[python end generated code: output=da39a3ee5e6b4b0d input=88f5dd06cd8e7a61]/
69	n/a
70	n/a	/* --- Globals ------------------------------------------------------------
71	n/a
72	n/a	NOTE: In the interpreter's initialization phase, some globals are currently
73	n/a	initialized dynamically as needed. In the process Unicode objects may
74	n/a	be created before the Unicode type is ready.
75	n/a
76	n/a	*/
77	n/a
78	n/a
79	n/a	#ifdef __cplusplus
80	n/a	extern "C" {
81	n/a	#endif
82	n/a
83	n/a	/* Maximum code point of Unicode 6.0: 0x10ffff (1,114,111) */
84	n/a	#define MAX_UNICODE 0x10ffff
85	n/a
86	n/a	#ifdef Py_DEBUG
87	n/a	# define _PyUnicode_CHECK(op) _PyUnicode_CheckConsistency(op, 0)
88	n/a	#else
89	n/a	# define _PyUnicode_CHECK(op) PyUnicode_Check(op)
90	n/a	#endif
91	n/a
92	n/a	#define _PyUnicode_UTF8(op) \
93	n/a	(((PyCompactUnicodeObject*)(op))->utf8)
94	n/a	#define PyUnicode_UTF8(op) \
95	n/a	(assert(_PyUnicode_CHECK(op)), \
96	n/a	assert(PyUnicode_IS_READY(op)), \
97	n/a	PyUnicode_IS_COMPACT_ASCII(op) ? \
98	n/a	((char)((PyASCIIObject)(op) + 1)) : \
99	n/a	_PyUnicode_UTF8(op))
100	n/a	#define _PyUnicode_UTF8_LENGTH(op) \
101	n/a	(((PyCompactUnicodeObject*)(op))->utf8_length)
102	n/a	#define PyUnicode_UTF8_LENGTH(op) \
103	n/a	(assert(_PyUnicode_CHECK(op)), \
104	n/a	assert(PyUnicode_IS_READY(op)), \
105	n/a	PyUnicode_IS_COMPACT_ASCII(op) ? \
106	n/a	((PyASCIIObject*)(op))->length : \
107	n/a	_PyUnicode_UTF8_LENGTH(op))
108	n/a	#define _PyUnicode_WSTR(op) \
109	n/a	(((PyASCIIObject*)(op))->wstr)
110	n/a	#define _PyUnicode_WSTR_LENGTH(op) \
111	n/a	(((PyCompactUnicodeObject*)(op))->wstr_length)
112	n/a	#define _PyUnicode_LENGTH(op) \
113	n/a	(((PyASCIIObject *)(op))->length)
114	n/a	#define _PyUnicode_STATE(op) \
115	n/a	(((PyASCIIObject *)(op))->state)
116	n/a	#define _PyUnicode_HASH(op) \
117	n/a	(((PyASCIIObject *)(op))->hash)
118	n/a	#define _PyUnicode_KIND(op) \
119	n/a	(assert(_PyUnicode_CHECK(op)), \
120	n/a	((PyASCIIObject *)(op))->state.kind)
121	n/a	#define _PyUnicode_GET_LENGTH(op) \
122	n/a	(assert(_PyUnicode_CHECK(op)), \
123	n/a	((PyASCIIObject *)(op))->length)
124	n/a	#define _PyUnicode_DATA_ANY(op) \
125	n/a	(((PyUnicodeObject*)(op))->data.any)
126	n/a
127	n/a	#undef PyUnicode_READY
128	n/a	#define PyUnicode_READY(op) \
129	n/a	(assert(_PyUnicode_CHECK(op)), \
130	n/a	(PyUnicode_IS_READY(op) ? \
131	n/a	0 : \
132	n/a	_PyUnicode_Ready(op)))
133	n/a
134	n/a	#define _PyUnicode_SHARE_UTF8(op) \
135	n/a	(assert(_PyUnicode_CHECK(op)), \
136	n/a	assert(!PyUnicode_IS_COMPACT_ASCII(op)), \
137	n/a	(_PyUnicode_UTF8(op) == PyUnicode_DATA(op)))
138	n/a	#define _PyUnicode_SHARE_WSTR(op) \
139	n/a	(assert(_PyUnicode_CHECK(op)), \
140	n/a	(_PyUnicode_WSTR(unicode) == PyUnicode_DATA(op)))
141	n/a
142	n/a	/* true if the Unicode object has an allocated UTF-8 memory block
143	n/a	(not shared with other data) */
144	n/a	#define _PyUnicode_HAS_UTF8_MEMORY(op) \
145	n/a	((!PyUnicode_IS_COMPACT_ASCII(op) \
146	n/a	&& _PyUnicode_UTF8(op) \
147	n/a	&& _PyUnicode_UTF8(op) != PyUnicode_DATA(op)))
148	n/a
149	n/a	/* true if the Unicode object has an allocated wstr memory block
150	n/a	(not shared with other data) */
151	n/a	#define _PyUnicode_HAS_WSTR_MEMORY(op) \
152	n/a	((_PyUnicode_WSTR(op) && \
153	n/a	(!PyUnicode_IS_READY(op) \|\| \
154	n/a	_PyUnicode_WSTR(op) != PyUnicode_DATA(op))))
155	n/a
156	n/a	/* Generic helper macro to convert characters of different types.
157	n/a	from_type and to_type have to be valid type names, begin and end
158	n/a	are pointers to the source characters which should be of type
159	n/a	"from_type ". to is a pointer of type "to_type " and points to the
160	n/a	buffer where the result characters are written to. */
161	n/a	#define _PyUnicode_CONVERT_BYTES(from_type, to_type, begin, end, to) \
162	n/a	do { \
163	n/a	to_type _to = (to_type )(to); \
164	n/a	const from_type _iter = (from_type )(begin); \
165	n/a	const from_type _end = (from_type )(end); \
166	n/a	Py_ssize_t n = (_end) - (_iter); \
167	n/a	const from_type *_unrolled_end = \
168	n/a	_iter + _Py_SIZE_ROUND_DOWN(n, 4); \
169	n/a	while (_iter < (_unrolled_end)) { \
170	n/a	_to[0] = (to_type) _iter[0]; \
171	n/a	_to[1] = (to_type) _iter[1]; \
172	n/a	_to[2] = (to_type) _iter[2]; \
173	n/a	_to[3] = (to_type) _iter[3]; \
174	n/a	_iter += 4; _to += 4; \
175	n/a	} \
176	n/a	while (_iter < (_end)) \
177	n/a	_to++ = (to_type) _iter++; \
178	n/a	} while (0)
179	n/a
180	n/a	#ifdef MS_WINDOWS
181	n/a	/* On Windows, overallocate by 50% is the best factor */
182	n/a	# define OVERALLOCATE_FACTOR 2
183	n/a	#else
184	n/a	/* On Linux, overallocate by 25% is the best factor */
185	n/a	# define OVERALLOCATE_FACTOR 4
186	n/a	#endif
187	n/a
188	n/a	/* This dictionary holds all interned unicode strings. Note that references
189	n/a	to strings in this dictionary are not counted in the string's ob_refcnt.
190	n/a	When the interned string reaches a refcnt of 0 the string deallocation
191	n/a	function will delete the reference from this dictionary.
192	n/a
193	n/a	Another way to look at this is that to say that the actual reference
194	n/a	count of a string is: s->ob_refcnt + (s->state ? 2 : 0)
195	n/a	*/
196	n/a	static PyObject *interned = NULL;
197	n/a
198	n/a	/* The empty Unicode object is shared to improve performance. */
199	n/a	static PyObject *unicode_empty = NULL;
200	n/a
201	n/a	#define _Py_INCREF_UNICODE_EMPTY() \
202	n/a	do { \
203	n/a	if (unicode_empty != NULL) \
204	n/a	Py_INCREF(unicode_empty); \
205	n/a	else { \
206	n/a	unicode_empty = PyUnicode_New(0, 0); \
207	n/a	if (unicode_empty != NULL) { \
208	n/a	Py_INCREF(unicode_empty); \
209	n/a	assert(_PyUnicode_CheckConsistency(unicode_empty, 1)); \
210	n/a	} \
211	n/a	} \
212	n/a	} while (0)
213	n/a
214	n/a	#define _Py_RETURN_UNICODE_EMPTY() \
215	n/a	do { \
216	n/a	_Py_INCREF_UNICODE_EMPTY(); \
217	n/a	return unicode_empty; \
218	n/a	} while (0)
219	n/a
220	n/a	/* Forward declaration */
221	n/a	static inline int
222	n/a	_PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch);
223	n/a
224	n/a	/* List of static strings. */
225	n/a	static _Py_Identifier *static_strings = NULL;
226	n/a
227	n/a	/* Single character Unicode strings in the Latin-1 range are being
228	n/a	shared as well. */
229	n/a	static PyObject *unicode_latin1[256] = {NULL};
230	n/a
231	n/a	/* Fast detection of the most frequent whitespace characters */
232	n/a	const unsigned char _Py_ascii_whitespace[] = {
233	n/a	0, 0, 0, 0, 0, 0, 0, 0,
234	n/a	/* case 0x0009: * CHARACTER TABULATION */
235	n/a	/* case 0x000A: * LINE FEED */
236	n/a	/* case 0x000B: * LINE TABULATION */
237	n/a	/* case 0x000C: * FORM FEED */
238	n/a	/* case 0x000D: * CARRIAGE RETURN */
239	n/a	0, 1, 1, 1, 1, 1, 0, 0,
240	n/a	0, 0, 0, 0, 0, 0, 0, 0,
241	n/a	/* case 0x001C: * FILE SEPARATOR */
242	n/a	/* case 0x001D: * GROUP SEPARATOR */
243	n/a	/* case 0x001E: * RECORD SEPARATOR */
244	n/a	/* case 0x001F: * UNIT SEPARATOR */
245	n/a	0, 0, 0, 0, 1, 1, 1, 1,
246	n/a	/* case 0x0020: * SPACE */
247	n/a	1, 0, 0, 0, 0, 0, 0, 0,
248	n/a	0, 0, 0, 0, 0, 0, 0, 0,
249	n/a	0, 0, 0, 0, 0, 0, 0, 0,
250	n/a	0, 0, 0, 0, 0, 0, 0, 0,
251	n/a
252	n/a	0, 0, 0, 0, 0, 0, 0, 0,
253	n/a	0, 0, 0, 0, 0, 0, 0, 0,
254	n/a	0, 0, 0, 0, 0, 0, 0, 0,
255	n/a	0, 0, 0, 0, 0, 0, 0, 0,
256	n/a	0, 0, 0, 0, 0, 0, 0, 0,
257	n/a	0, 0, 0, 0, 0, 0, 0, 0,
258	n/a	0, 0, 0, 0, 0, 0, 0, 0,
259	n/a	0, 0, 0, 0, 0, 0, 0, 0
260	n/a	};
261	n/a
262	n/a	/* forward */
263	n/a	static PyUnicodeObject *_PyUnicode_New(Py_ssize_t length);
264	n/a	static PyObject* get_latin1_char(unsigned char ch);
265	n/a	static int unicode_modifiable(PyObject *unicode);
266	n/a
267	n/a
268	n/a	static PyObject *
269	n/a	_PyUnicode_FromUCS1(const Py_UCS1 *s, Py_ssize_t size);
270	n/a	static PyObject *
271	n/a	_PyUnicode_FromUCS2(const Py_UCS2 *s, Py_ssize_t size);
272	n/a	static PyObject *
273	n/a	_PyUnicode_FromUCS4(const Py_UCS4 *s, Py_ssize_t size);
274	n/a
275	n/a	static PyObject *
276	n/a	unicode_encode_call_errorhandler(const char *errors,
277	n/a	PyObject *errorHandler,const char encoding, const char *reason,
278	n/a	PyObject unicode, PyObject *exceptionObject,
279	n/a	Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos);
280	n/a
281	n/a	static void
282	n/a	raise_encode_exception(PyObject **exceptionObject,
283	n/a	const char *encoding,
284	n/a	PyObject *unicode,
285	n/a	Py_ssize_t startpos, Py_ssize_t endpos,
286	n/a	const char *reason);
287	n/a
288	n/a	/* Same for linebreaks */
289	n/a	static const unsigned char ascii_linebreak[] = {
290	n/a	0, 0, 0, 0, 0, 0, 0, 0,
291	n/a	/* 0x000A, * LINE FEED */
292	n/a	/* 0x000B, * LINE TABULATION */
293	n/a	/* 0x000C, * FORM FEED */
294	n/a	/* 0x000D, * CARRIAGE RETURN */
295	n/a	0, 0, 1, 1, 1, 1, 0, 0,
296	n/a	0, 0, 0, 0, 0, 0, 0, 0,
297	n/a	/* 0x001C, * FILE SEPARATOR */
298	n/a	/* 0x001D, * GROUP SEPARATOR */
299	n/a	/* 0x001E, * RECORD SEPARATOR */
300	n/a	0, 0, 0, 0, 1, 1, 1, 0,
301	n/a	0, 0, 0, 0, 0, 0, 0, 0,
302	n/a	0, 0, 0, 0, 0, 0, 0, 0,
303	n/a	0, 0, 0, 0, 0, 0, 0, 0,
304	n/a	0, 0, 0, 0, 0, 0, 0, 0,
305	n/a
306	n/a	0, 0, 0, 0, 0, 0, 0, 0,
307	n/a	0, 0, 0, 0, 0, 0, 0, 0,
308	n/a	0, 0, 0, 0, 0, 0, 0, 0,
309	n/a	0, 0, 0, 0, 0, 0, 0, 0,
310	n/a	0, 0, 0, 0, 0, 0, 0, 0,
311	n/a	0, 0, 0, 0, 0, 0, 0, 0,
312	n/a	0, 0, 0, 0, 0, 0, 0, 0,
313	n/a	0, 0, 0, 0, 0, 0, 0, 0
314	n/a	};
315	n/a
316	n/a	static int convert_uc(PyObject obj, void addr);
317	n/a
318	n/a	#include "clinic/unicodeobject.c.h"
319	n/a
320	n/a	typedef enum {
321	n/a	_Py_ERROR_UNKNOWN=0,
322	n/a	_Py_ERROR_STRICT,
323	n/a	_Py_ERROR_SURROGATEESCAPE,
324	n/a	_Py_ERROR_REPLACE,
325	n/a	_Py_ERROR_IGNORE,
326	n/a	_Py_ERROR_BACKSLASHREPLACE,
327	n/a	_Py_ERROR_SURROGATEPASS,
328	n/a	_Py_ERROR_XMLCHARREFREPLACE,
329	n/a	_Py_ERROR_OTHER
330	n/a	} _Py_error_handler;
331	n/a
332	n/a	static _Py_error_handler
333	n/a	get_error_handler(const char *errors)
334	n/a	{
335	n/a	if (errors == NULL \|\| strcmp(errors, "strict") == 0) {
336	n/a	return _Py_ERROR_STRICT;
337	n/a	}
338	n/a	if (strcmp(errors, "surrogateescape") == 0) {
339	n/a	return _Py_ERROR_SURROGATEESCAPE;
340	n/a	}
341	n/a	if (strcmp(errors, "replace") == 0) {
342	n/a	return _Py_ERROR_REPLACE;
343	n/a	}
344	n/a	if (strcmp(errors, "ignore") == 0) {
345	n/a	return _Py_ERROR_IGNORE;
346	n/a	}
347	n/a	if (strcmp(errors, "backslashreplace") == 0) {
348	n/a	return _Py_ERROR_BACKSLASHREPLACE;
349	n/a	}
350	n/a	if (strcmp(errors, "surrogatepass") == 0) {
351	n/a	return _Py_ERROR_SURROGATEPASS;
352	n/a	}
353	n/a	if (strcmp(errors, "xmlcharrefreplace") == 0) {
354	n/a	return _Py_ERROR_XMLCHARREFREPLACE;
355	n/a	}
356	n/a	return _Py_ERROR_OTHER;
357	n/a	}
358	n/a
359	n/a	/* The max unicode value is always 0x10FFFF while using the PEP-393 API.
360	n/a	This function is kept for backward compatibility with the old API. */
361	n/a	Py_UNICODE
362	n/a	PyUnicode_GetMax(void)
363	n/a	{
364	n/a	#ifdef Py_UNICODE_WIDE
365	n/a	return 0x10FFFF;
366	n/a	#else
367	n/a	/* This is actually an illegal character, so it should
368	n/a	not be passed to unichr. */
369	n/a	return 0xFFFF;
370	n/a	#endif
371	n/a	}
372	n/a
373	n/a	#ifdef Py_DEBUG
374	n/a	int
375	n/a	_PyUnicode_CheckConsistency(PyObject *op, int check_content)
376	n/a	{
377	n/a	PyASCIIObject *ascii;
378	n/a	unsigned int kind;
379	n/a
380	n/a	assert(PyUnicode_Check(op));
381	n/a
382	n/a	ascii = (PyASCIIObject *)op;
383	n/a	kind = ascii->state.kind;
384	n/a
385	n/a	if (ascii->state.ascii == 1 && ascii->state.compact == 1) {
386	n/a	assert(kind == PyUnicode_1BYTE_KIND);
387	n/a	assert(ascii->state.ready == 1);
388	n/a	}
389	n/a	else {
390	n/a	PyCompactUnicodeObject compact = (PyCompactUnicodeObject )op;
391	n/a	void *data;
392	n/a
393	n/a	if (ascii->state.compact == 1) {
394	n/a	data = compact + 1;
395	n/a	assert(kind == PyUnicode_1BYTE_KIND
396	n/a	\|\| kind == PyUnicode_2BYTE_KIND
397	n/a	\|\| kind == PyUnicode_4BYTE_KIND);
398	n/a	assert(ascii->state.ascii == 0);
399	n/a	assert(ascii->state.ready == 1);
400	n/a	assert (compact->utf8 != data);
401	n/a	}
402	n/a	else {
403	n/a	PyUnicodeObject unicode = (PyUnicodeObject )op;
404	n/a
405	n/a	data = unicode->data.any;
406	n/a	if (kind == PyUnicode_WCHAR_KIND) {
407	n/a	assert(ascii->length == 0);
408	n/a	assert(ascii->hash == -1);
409	n/a	assert(ascii->state.compact == 0);
410	n/a	assert(ascii->state.ascii == 0);
411	n/a	assert(ascii->state.ready == 0);
412	n/a	assert(ascii->state.interned == SSTATE_NOT_INTERNED);
413	n/a	assert(ascii->wstr != NULL);
414	n/a	assert(data == NULL);
415	n/a	assert(compact->utf8 == NULL);
416	n/a	}
417	n/a	else {
418	n/a	assert(kind == PyUnicode_1BYTE_KIND
419	n/a	\|\| kind == PyUnicode_2BYTE_KIND
420	n/a	\|\| kind == PyUnicode_4BYTE_KIND);
421	n/a	assert(ascii->state.compact == 0);
422	n/a	assert(ascii->state.ready == 1);
423	n/a	assert(data != NULL);
424	n/a	if (ascii->state.ascii) {
425	n/a	assert (compact->utf8 == data);
426	n/a	assert (compact->utf8_length == ascii->length);
427	n/a	}
428	n/a	else
429	n/a	assert (compact->utf8 != data);
430	n/a	}
431	n/a	}
432	n/a	if (kind != PyUnicode_WCHAR_KIND) {
433	n/a	if (
434	n/a	#if SIZEOF_WCHAR_T == 2
435	n/a	kind == PyUnicode_2BYTE_KIND
436	n/a	#else
437	n/a	kind == PyUnicode_4BYTE_KIND
438	n/a	#endif
439	n/a	)
440	n/a	{
441	n/a	assert(ascii->wstr == data);
442	n/a	assert(compact->wstr_length == ascii->length);
443	n/a	} else
444	n/a	assert(ascii->wstr != data);
445	n/a	}
446	n/a
447	n/a	if (compact->utf8 == NULL)
448	n/a	assert(compact->utf8_length == 0);
449	n/a	if (ascii->wstr == NULL)
450	n/a	assert(compact->wstr_length == 0);
451	n/a	}
452	n/a	/* check that the best kind is used */
453	n/a	if (check_content && kind != PyUnicode_WCHAR_KIND)
454	n/a	{
455	n/a	Py_ssize_t i;
456	n/a	Py_UCS4 maxchar = 0;
457	n/a	void *data;
458	n/a	Py_UCS4 ch;
459	n/a
460	n/a	data = PyUnicode_DATA(ascii);
461	n/a	for (i=0; i < ascii->length; i++)
462	n/a	{
463	n/a	ch = PyUnicode_READ(kind, data, i);
464	n/a	if (ch > maxchar)
465	n/a	maxchar = ch;
466	n/a	}
467	n/a	if (kind == PyUnicode_1BYTE_KIND) {
468	n/a	if (ascii->state.ascii == 0) {
469	n/a	assert(maxchar >= 128);
470	n/a	assert(maxchar <= 255);
471	n/a	}
472	n/a	else
473	n/a	assert(maxchar < 128);
474	n/a	}
475	n/a	else if (kind == PyUnicode_2BYTE_KIND) {
476	n/a	assert(maxchar >= 0x100);
477	n/a	assert(maxchar <= 0xFFFF);
478	n/a	}
479	n/a	else {
480	n/a	assert(maxchar >= 0x10000);
481	n/a	assert(maxchar <= MAX_UNICODE);
482	n/a	}
483	n/a	assert(PyUnicode_READ(kind, data, ascii->length) == 0);
484	n/a	}
485	n/a	return 1;
486	n/a	}
487	n/a	#endif
488	n/a
489	n/a	static PyObject*
490	n/a	unicode_result_wchar(PyObject *unicode)
491	n/a	{
492	n/a	#ifndef Py_DEBUG
493	n/a	Py_ssize_t len;
494	n/a
495	n/a	len = _PyUnicode_WSTR_LENGTH(unicode);
496	n/a	if (len == 0) {
497	n/a	Py_DECREF(unicode);
498	n/a	_Py_RETURN_UNICODE_EMPTY();
499	n/a	}
500	n/a
501	n/a	if (len == 1) {
502	n/a	wchar_t ch = _PyUnicode_WSTR(unicode)[0];
503	n/a	if ((Py_UCS4)ch < 256) {
504	n/a	PyObject *latin1_char = get_latin1_char((unsigned char)ch);
505	n/a	Py_DECREF(unicode);
506	n/a	return latin1_char;
507	n/a	}
508	n/a	}
509	n/a
510	n/a	if (_PyUnicode_Ready(unicode) < 0) {
511	n/a	Py_DECREF(unicode);
512	n/a	return NULL;
513	n/a	}
514	n/a	#else
515	n/a	assert(Py_REFCNT(unicode) == 1);
516	n/a
517	n/a	/* don't make the result ready in debug mode to ensure that the caller
518	n/a	makes the string ready before using it */
519	n/a	assert(_PyUnicode_CheckConsistency(unicode, 1));
520	n/a	#endif
521	n/a	return unicode;
522	n/a	}
523	n/a
524	n/a	static PyObject*
525	n/a	unicode_result_ready(PyObject *unicode)
526	n/a	{
527	n/a	Py_ssize_t length;
528	n/a
529	n/a	length = PyUnicode_GET_LENGTH(unicode);
530	n/a	if (length == 0) {
531	n/a	if (unicode != unicode_empty) {
532	n/a	Py_DECREF(unicode);
533	n/a	_Py_RETURN_UNICODE_EMPTY();
534	n/a	}
535	n/a	return unicode_empty;
536	n/a	}
537	n/a
538	n/a	if (length == 1) {
539	n/a	void *data = PyUnicode_DATA(unicode);
540	n/a	int kind = PyUnicode_KIND(unicode);
541	n/a	Py_UCS4 ch = PyUnicode_READ(kind, data, 0);
542	n/a	if (ch < 256) {
543	n/a	PyObject *latin1_char = unicode_latin1[ch];
544	n/a	if (latin1_char != NULL) {
545	n/a	if (unicode != latin1_char) {
546	n/a	Py_INCREF(latin1_char);
547	n/a	Py_DECREF(unicode);
548	n/a	}
549	n/a	return latin1_char;
550	n/a	}
551	n/a	else {
552	n/a	assert(_PyUnicode_CheckConsistency(unicode, 1));
553	n/a	Py_INCREF(unicode);
554	n/a	unicode_latin1[ch] = unicode;
555	n/a	return unicode;
556	n/a	}
557	n/a	}
558	n/a	}
559	n/a
560	n/a	assert(_PyUnicode_CheckConsistency(unicode, 1));
561	n/a	return unicode;
562	n/a	}
563	n/a
564	n/a	static PyObject*
565	n/a	unicode_result(PyObject *unicode)
566	n/a	{
567	n/a	assert(_PyUnicode_CHECK(unicode));
568	n/a	if (PyUnicode_IS_READY(unicode))
569	n/a	return unicode_result_ready(unicode);
570	n/a	else
571	n/a	return unicode_result_wchar(unicode);
572	n/a	}
573	n/a
574	n/a	static PyObject*
575	n/a	unicode_result_unchanged(PyObject *unicode)
576	n/a	{
577	n/a	if (PyUnicode_CheckExact(unicode)) {
578	n/a	if (PyUnicode_READY(unicode) == -1)
579	n/a	return NULL;
580	n/a	Py_INCREF(unicode);
581	n/a	return unicode;
582	n/a	}
583	n/a	else
584	n/a	/* Subtype -- return genuine unicode string with the same value. */
585	n/a	return _PyUnicode_Copy(unicode);
586	n/a	}
587	n/a
588	n/a	/* Implementation of the "backslashreplace" error handler for 8-bit encodings:
589	n/a	ASCII, Latin1, UTF-8, etc. */
590	n/a	static char*
591	n/a	backslashreplace(_PyBytesWriter writer, char str,
592	n/a	PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend)
593	n/a	{
594	n/a	Py_ssize_t size, i;
595	n/a	Py_UCS4 ch;
596	n/a	enum PyUnicode_Kind kind;
597	n/a	void *data;
598	n/a
599	n/a	assert(PyUnicode_IS_READY(unicode));
600	n/a	kind = PyUnicode_KIND(unicode);
601	n/a	data = PyUnicode_DATA(unicode);
602	n/a
603	n/a	size = 0;
604	n/a	/* determine replacement size */
605	n/a	for (i = collstart; i < collend; ++i) {
606	n/a	Py_ssize_t incr;
607	n/a
608	n/a	ch = PyUnicode_READ(kind, data, i);
609	n/a	if (ch < 0x100)
610	n/a	incr = 2+2;
611	n/a	else if (ch < 0x10000)
612	n/a	incr = 2+4;
613	n/a	else {
614	n/a	assert(ch <= MAX_UNICODE);
615	n/a	incr = 2+8;
616	n/a	}
617	n/a	if (size > PY_SSIZE_T_MAX - incr) {
618	n/a	PyErr_SetString(PyExc_OverflowError,
619	n/a	"encoded result is too long for a Python string");
620	n/a	return NULL;
621	n/a	}
622	n/a	size += incr;
623	n/a	}
624	n/a
625	n/a	str = _PyBytesWriter_Prepare(writer, str, size);
626	n/a	if (str == NULL)
627	n/a	return NULL;
628	n/a
629	n/a	/* generate replacement */
630	n/a	for (i = collstart; i < collend; ++i) {
631	n/a	ch = PyUnicode_READ(kind, data, i);
632	n/a	*str++ = '\\';
633	n/a	if (ch >= 0x00010000) {
634	n/a	*str++ = 'U';
635	n/a	*str++ = Py_hexdigits[(ch>>28)&0xf];
636	n/a	*str++ = Py_hexdigits[(ch>>24)&0xf];
637	n/a	*str++ = Py_hexdigits[(ch>>20)&0xf];
638	n/a	*str++ = Py_hexdigits[(ch>>16)&0xf];
639	n/a	*str++ = Py_hexdigits[(ch>>12)&0xf];
640	n/a	*str++ = Py_hexdigits[(ch>>8)&0xf];
641	n/a	}
642	n/a	else if (ch >= 0x100) {
643	n/a	*str++ = 'u';
644	n/a	*str++ = Py_hexdigits[(ch>>12)&0xf];
645	n/a	*str++ = Py_hexdigits[(ch>>8)&0xf];
646	n/a	}
647	n/a	else
648	n/a	*str++ = 'x';
649	n/a	*str++ = Py_hexdigits[(ch>>4)&0xf];
650	n/a	*str++ = Py_hexdigits[ch&0xf];
651	n/a	}
652	n/a	return str;
653	n/a	}
654	n/a
655	n/a	/* Implementation of the "xmlcharrefreplace" error handler for 8-bit encodings:
656	n/a	ASCII, Latin1, UTF-8, etc. */
657	n/a	static char*
658	n/a	xmlcharrefreplace(_PyBytesWriter writer, char str,
659	n/a	PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend)
660	n/a	{
661	n/a	Py_ssize_t size, i;
662	n/a	Py_UCS4 ch;
663	n/a	enum PyUnicode_Kind kind;
664	n/a	void *data;
665	n/a
666	n/a	assert(PyUnicode_IS_READY(unicode));
667	n/a	kind = PyUnicode_KIND(unicode);
668	n/a	data = PyUnicode_DATA(unicode);
669	n/a
670	n/a	size = 0;
671	n/a	/* determine replacement size */
672	n/a	for (i = collstart; i < collend; ++i) {
673	n/a	Py_ssize_t incr;
674	n/a
675	n/a	ch = PyUnicode_READ(kind, data, i);
676	n/a	if (ch < 10)
677	n/a	incr = 2+1+1;
678	n/a	else if (ch < 100)
679	n/a	incr = 2+2+1;
680	n/a	else if (ch < 1000)
681	n/a	incr = 2+3+1;
682	n/a	else if (ch < 10000)
683	n/a	incr = 2+4+1;
684	n/a	else if (ch < 100000)
685	n/a	incr = 2+5+1;
686	n/a	else if (ch < 1000000)
687	n/a	incr = 2+6+1;
688	n/a	else {
689	n/a	assert(ch <= MAX_UNICODE);
690	n/a	incr = 2+7+1;
691	n/a	}
692	n/a	if (size > PY_SSIZE_T_MAX - incr) {
693	n/a	PyErr_SetString(PyExc_OverflowError,
694	n/a	"encoded result is too long for a Python string");
695	n/a	return NULL;
696	n/a	}
697	n/a	size += incr;
698	n/a	}
699	n/a
700	n/a	str = _PyBytesWriter_Prepare(writer, str, size);
701	n/a	if (str == NULL)
702	n/a	return NULL;
703	n/a
704	n/a	/* generate replacement */
705	n/a	for (i = collstart; i < collend; ++i) {
706	n/a	str += sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i));
707	n/a	}
708	n/a	return str;
709	n/a	}
710	n/a
711	n/a	/* --- Bloom Filters ----------------------------------------------------- */
712	n/a
713	n/a	/* stuff to implement simple "bloom filters" for Unicode characters.
714	n/a	to keep things simple, we use a single bitmask, using the least 5
715	n/a	bits from each unicode characters as the bit index. */
716	n/a
717	n/a	/* the linebreak mask is set up by Unicode_Init below */
718	n/a
719	n/a	#if LONG_BIT >= 128
720	n/a	#define BLOOM_WIDTH 128
721	n/a	#elif LONG_BIT >= 64
722	n/a	#define BLOOM_WIDTH 64
723	n/a	#elif LONG_BIT >= 32
724	n/a	#define BLOOM_WIDTH 32
725	n/a	#else
726	n/a	#error "LONG_BIT is smaller than 32"
727	n/a	#endif
728	n/a
729	n/a	#define BLOOM_MASK unsigned long
730	n/a
731	n/a	static BLOOM_MASK bloom_linebreak = ~(BLOOM_MASK)0;
732	n/a
733	n/a	#define BLOOM(mask, ch) ((mask & (1UL << ((ch) & (BLOOM_WIDTH - 1)))))
734	n/a
735	n/a	#define BLOOM_LINEBREAK(ch) \
736	n/a	((ch) < 128U ? ascii_linebreak[(ch)] : \
737	n/a	(BLOOM(bloom_linebreak, (ch)) && Py_UNICODE_ISLINEBREAK(ch)))
738	n/a
739	n/a	static inline BLOOM_MASK
740	n/a	make_bloom_mask(int kind, void* ptr, Py_ssize_t len)
741	n/a	{
742	n/a	#define BLOOM_UPDATE(TYPE, MASK, PTR, LEN) \
743	n/a	do { \
744	n/a	TYPE data = (TYPE )PTR; \
745	n/a	TYPE *end = data + LEN; \
746	n/a	Py_UCS4 ch; \
747	n/a	for (; data != end; data++) { \
748	n/a	ch = *data; \
749	n/a	MASK \|= (1UL << (ch & (BLOOM_WIDTH - 1))); \
750	n/a	} \
751	n/a	break; \
752	n/a	} while (0)
753	n/a
754	n/a	/* calculate simple bloom-style bitmask for a given unicode string */
755	n/a
756	n/a	BLOOM_MASK mask;
757	n/a
758	n/a	mask = 0;
759	n/a	switch (kind) {
760	n/a	case PyUnicode_1BYTE_KIND:
761	n/a	BLOOM_UPDATE(Py_UCS1, mask, ptr, len);
762	n/a	break;
763	n/a	case PyUnicode_2BYTE_KIND:
764	n/a	BLOOM_UPDATE(Py_UCS2, mask, ptr, len);
765	n/a	break;
766	n/a	case PyUnicode_4BYTE_KIND:
767	n/a	BLOOM_UPDATE(Py_UCS4, mask, ptr, len);
768	n/a	break;
769	n/a	default:
770	n/a	assert(0);
771	n/a	}
772	n/a	return mask;
773	n/a
774	n/a	#undef BLOOM_UPDATE
775	n/a	}
776	n/a
777	n/a	static int
778	n/a	ensure_unicode(PyObject *obj)
779	n/a	{
780	n/a	if (!PyUnicode_Check(obj)) {
781	n/a	PyErr_Format(PyExc_TypeError,
782	n/a	"must be str, not %.100s",
783	n/a	Py_TYPE(obj)->tp_name);
784	n/a	return -1;
785	n/a	}
786	n/a	return PyUnicode_READY(obj);
787	n/a	}
788	n/a
789	n/a	/* Compilation of templated routines */
790	n/a
791	n/a	#include "stringlib/asciilib.h"
792	n/a	#include "stringlib/fastsearch.h"
793	n/a	#include "stringlib/partition.h"
794	n/a	#include "stringlib/split.h"
795	n/a	#include "stringlib/count.h"
796	n/a	#include "stringlib/find.h"
797	n/a	#include "stringlib/find_max_char.h"
798	n/a	#include "stringlib/localeutil.h"
799	n/a	#include "stringlib/undef.h"
800	n/a
801	n/a	#include "stringlib/ucs1lib.h"
802	n/a	#include "stringlib/fastsearch.h"
803	n/a	#include "stringlib/partition.h"
804	n/a	#include "stringlib/split.h"
805	n/a	#include "stringlib/count.h"
806	n/a	#include "stringlib/find.h"
807	n/a	#include "stringlib/replace.h"
808	n/a	#include "stringlib/find_max_char.h"
809	n/a	#include "stringlib/localeutil.h"
810	n/a	#include "stringlib/undef.h"
811	n/a
812	n/a	#include "stringlib/ucs2lib.h"
813	n/a	#include "stringlib/fastsearch.h"
814	n/a	#include "stringlib/partition.h"
815	n/a	#include "stringlib/split.h"
816	n/a	#include "stringlib/count.h"
817	n/a	#include "stringlib/find.h"
818	n/a	#include "stringlib/replace.h"
819	n/a	#include "stringlib/find_max_char.h"
820	n/a	#include "stringlib/localeutil.h"
821	n/a	#include "stringlib/undef.h"
822	n/a
823	n/a	#include "stringlib/ucs4lib.h"
824	n/a	#include "stringlib/fastsearch.h"
825	n/a	#include "stringlib/partition.h"
826	n/a	#include "stringlib/split.h"
827	n/a	#include "stringlib/count.h"
828	n/a	#include "stringlib/find.h"
829	n/a	#include "stringlib/replace.h"
830	n/a	#include "stringlib/find_max_char.h"
831	n/a	#include "stringlib/localeutil.h"
832	n/a	#include "stringlib/undef.h"
833	n/a
834	n/a	#include "stringlib/unicodedefs.h"
835	n/a	#include "stringlib/fastsearch.h"
836	n/a	#include "stringlib/count.h"
837	n/a	#include "stringlib/find.h"
838	n/a	#include "stringlib/undef.h"
839	n/a
840	n/a	/* --- Unicode Object ----------------------------------------------------- */
841	n/a
842	n/a	static PyObject *
843	n/a	fixup(PyObject self, Py_UCS4 (fixfct)(PyObject *s));
844	n/a
845	n/a	static inline Py_ssize_t
846	n/a	findchar(const void *s, int kind,
847	n/a	Py_ssize_t size, Py_UCS4 ch,
848	n/a	int direction)
849	n/a	{
850	n/a	switch (kind) {
851	n/a	case PyUnicode_1BYTE_KIND:
852	n/a	if ((Py_UCS1) ch != ch)
853	n/a	return -1;
854	n/a	if (direction > 0)
855	n/a	return ucs1lib_find_char((Py_UCS1 *) s, size, (Py_UCS1) ch);
856	n/a	else
857	n/a	return ucs1lib_rfind_char((Py_UCS1 *) s, size, (Py_UCS1) ch);
858	n/a	case PyUnicode_2BYTE_KIND:
859	n/a	if ((Py_UCS2) ch != ch)
860	n/a	return -1;
861	n/a	if (direction > 0)
862	n/a	return ucs2lib_find_char((Py_UCS2 *) s, size, (Py_UCS2) ch);
863	n/a	else
864	n/a	return ucs2lib_rfind_char((Py_UCS2 *) s, size, (Py_UCS2) ch);
865	n/a	case PyUnicode_4BYTE_KIND:
866	n/a	if (direction > 0)
867	n/a	return ucs4lib_find_char((Py_UCS4 *) s, size, ch);
868	n/a	else
869	n/a	return ucs4lib_rfind_char((Py_UCS4 *) s, size, ch);
870	n/a	default:
871	n/a	assert(0);
872	n/a	return -1;
873	n/a	}
874	n/a	}
875	n/a
876	n/a	#ifdef Py_DEBUG
877	n/a	/* Fill the data of a Unicode string with invalid characters to detect bugs
878	n/a	earlier.
879	n/a
880	n/a	_PyUnicode_CheckConsistency(str, 1) detects invalid characters, at least for
881	n/a	ASCII and UCS-4 strings. U+00FF is invalid in ASCII and U+FFFFFFFF is an
882	n/a	invalid character in Unicode 6.0. */
883	n/a	static void
884	n/a	unicode_fill_invalid(PyObject *unicode, Py_ssize_t old_length)
885	n/a	{
886	n/a	int kind = PyUnicode_KIND(unicode);
887	n/a	Py_UCS1 *data = PyUnicode_1BYTE_DATA(unicode);
888	n/a	Py_ssize_t length = _PyUnicode_LENGTH(unicode);
889	n/a	if (length <= old_length)
890	n/a	return;
891	n/a	memset(data + old_length * kind, 0xff, (length - old_length) * kind);
892	n/a	}
893	n/a	#endif
894	n/a
895	n/a	static PyObject*
896	n/a	resize_compact(PyObject *unicode, Py_ssize_t length)
897	n/a	{
898	n/a	Py_ssize_t char_size;
899	n/a	Py_ssize_t struct_size;
900	n/a	Py_ssize_t new_size;
901	n/a	int share_wstr;
902	n/a	PyObject *new_unicode;
903	n/a	#ifdef Py_DEBUG
904	n/a	Py_ssize_t old_length = _PyUnicode_LENGTH(unicode);
905	n/a	#endif
906	n/a
907	n/a	assert(unicode_modifiable(unicode));
908	n/a	assert(PyUnicode_IS_READY(unicode));
909	n/a	assert(PyUnicode_IS_COMPACT(unicode));
910	n/a
911	n/a	char_size = PyUnicode_KIND(unicode);
912	n/a	if (PyUnicode_IS_ASCII(unicode))
913	n/a	struct_size = sizeof(PyASCIIObject);
914	n/a	else
915	n/a	struct_size = sizeof(PyCompactUnicodeObject);
916	n/a	share_wstr = _PyUnicode_SHARE_WSTR(unicode);
917	n/a
918	n/a	if (length > ((PY_SSIZE_T_MAX - struct_size) / char_size - 1)) {
919	n/a	PyErr_NoMemory();
920	n/a	return NULL;
921	n/a	}
922	n/a	new_size = (struct_size + (length + 1) * char_size);
923	n/a
924	n/a	if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) {
925	n/a	PyObject_DEL(_PyUnicode_UTF8(unicode));
926	n/a	_PyUnicode_UTF8(unicode) = NULL;
927	n/a	_PyUnicode_UTF8_LENGTH(unicode) = 0;
928	n/a	}
929	n/a	_Py_DEC_REFTOTAL;
930	n/a	_Py_ForgetReference(unicode);
931	n/a
932	n/a	new_unicode = (PyObject *)PyObject_REALLOC(unicode, new_size);
933	n/a	if (new_unicode == NULL) {
934	n/a	_Py_NewReference(unicode);
935	n/a	PyErr_NoMemory();
936	n/a	return NULL;
937	n/a	}
938	n/a	unicode = new_unicode;
939	n/a	_Py_NewReference(unicode);
940	n/a
941	n/a	_PyUnicode_LENGTH(unicode) = length;
942	n/a	if (share_wstr) {
943	n/a	_PyUnicode_WSTR(unicode) = PyUnicode_DATA(unicode);
944	n/a	if (!PyUnicode_IS_ASCII(unicode))
945	n/a	_PyUnicode_WSTR_LENGTH(unicode) = length;
946	n/a	}
947	n/a	else if (_PyUnicode_HAS_WSTR_MEMORY(unicode)) {
948	n/a	PyObject_DEL(_PyUnicode_WSTR(unicode));
949	n/a	_PyUnicode_WSTR(unicode) = NULL;
950	n/a	if (!PyUnicode_IS_ASCII(unicode))
951	n/a	_PyUnicode_WSTR_LENGTH(unicode) = 0;
952	n/a	}
953	n/a	#ifdef Py_DEBUG
954	n/a	unicode_fill_invalid(unicode, old_length);
955	n/a	#endif
956	n/a	PyUnicode_WRITE(PyUnicode_KIND(unicode), PyUnicode_DATA(unicode),
957	n/a	length, 0);
958	n/a	assert(_PyUnicode_CheckConsistency(unicode, 0));
959	n/a	return unicode;
960	n/a	}
961	n/a
962	n/a	static int
963	n/a	resize_inplace(PyObject *unicode, Py_ssize_t length)
964	n/a	{
965	n/a	wchar_t *wstr;
966	n/a	Py_ssize_t new_size;
967	n/a	assert(!PyUnicode_IS_COMPACT(unicode));
968	n/a	assert(Py_REFCNT(unicode) == 1);
969	n/a
970	n/a	if (PyUnicode_IS_READY(unicode)) {
971	n/a	Py_ssize_t char_size;
972	n/a	int share_wstr, share_utf8;
973	n/a	void *data;
974	n/a	#ifdef Py_DEBUG
975	n/a	Py_ssize_t old_length = _PyUnicode_LENGTH(unicode);
976	n/a	#endif
977	n/a
978	n/a	data = _PyUnicode_DATA_ANY(unicode);
979	n/a	char_size = PyUnicode_KIND(unicode);
980	n/a	share_wstr = _PyUnicode_SHARE_WSTR(unicode);
981	n/a	share_utf8 = _PyUnicode_SHARE_UTF8(unicode);
982	n/a
983	n/a	if (length > (PY_SSIZE_T_MAX / char_size - 1)) {
984	n/a	PyErr_NoMemory();
985	n/a	return -1;
986	n/a	}
987	n/a	new_size = (length + 1) * char_size;
988	n/a
989	n/a	if (!share_utf8 && _PyUnicode_HAS_UTF8_MEMORY(unicode))
990	n/a	{
991	n/a	PyObject_DEL(_PyUnicode_UTF8(unicode));
992	n/a	_PyUnicode_UTF8(unicode) = NULL;
993	n/a	_PyUnicode_UTF8_LENGTH(unicode) = 0;
994	n/a	}
995	n/a
996	n/a	data = (PyObject *)PyObject_REALLOC(data, new_size);
997	n/a	if (data == NULL) {
998	n/a	PyErr_NoMemory();
999	n/a	return -1;
1000	n/a	}
1001	n/a	_PyUnicode_DATA_ANY(unicode) = data;
1002	n/a	if (share_wstr) {
1003	n/a	_PyUnicode_WSTR(unicode) = data;
1004	n/a	_PyUnicode_WSTR_LENGTH(unicode) = length;
1005	n/a	}
1006	n/a	if (share_utf8) {
1007	n/a	_PyUnicode_UTF8(unicode) = data;
1008	n/a	_PyUnicode_UTF8_LENGTH(unicode) = length;
1009	n/a	}
1010	n/a	_PyUnicode_LENGTH(unicode) = length;
1011	n/a	PyUnicode_WRITE(PyUnicode_KIND(unicode), data, length, 0);
1012	n/a	#ifdef Py_DEBUG
1013	n/a	unicode_fill_invalid(unicode, old_length);
1014	n/a	#endif
1015	n/a	if (share_wstr \|\| _PyUnicode_WSTR(unicode) == NULL) {
1016	n/a	assert(_PyUnicode_CheckConsistency(unicode, 0));
1017	n/a	return 0;
1018	n/a	}
1019	n/a	}
1020	n/a	assert(_PyUnicode_WSTR(unicode) != NULL);
1021	n/a
1022	n/a	/* check for integer overflow */
1023	n/a	if (length > PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(wchar_t) - 1) {
1024	n/a	PyErr_NoMemory();
1025	n/a	return -1;
1026	n/a	}
1027	n/a	new_size = sizeof(wchar_t) * (length + 1);
1028	n/a	wstr = _PyUnicode_WSTR(unicode);
1029	n/a	wstr = PyObject_REALLOC(wstr, new_size);
1030	n/a	if (!wstr) {
1031	n/a	PyErr_NoMemory();
1032	n/a	return -1;
1033	n/a	}
1034	n/a	_PyUnicode_WSTR(unicode) = wstr;
1035	n/a	_PyUnicode_WSTR(unicode)[length] = 0;
1036	n/a	_PyUnicode_WSTR_LENGTH(unicode) = length;
1037	n/a	assert(_PyUnicode_CheckConsistency(unicode, 0));
1038	n/a	return 0;
1039	n/a	}
1040	n/a
1041	n/a	static PyObject*
1042	n/a	resize_copy(PyObject *unicode, Py_ssize_t length)
1043	n/a	{
1044	n/a	Py_ssize_t copy_length;
1045	n/a	if (_PyUnicode_KIND(unicode) != PyUnicode_WCHAR_KIND) {
1046	n/a	PyObject *copy;
1047	n/a
1048	n/a	assert(PyUnicode_IS_READY(unicode));
1049	n/a
1050	n/a	copy = PyUnicode_New(length, PyUnicode_MAX_CHAR_VALUE(unicode));
1051	n/a	if (copy == NULL)
1052	n/a	return NULL;
1053	n/a
1054	n/a	copy_length = Py_MIN(length, PyUnicode_GET_LENGTH(unicode));
1055	n/a	_PyUnicode_FastCopyCharacters(copy, 0, unicode, 0, copy_length);
1056	n/a	return copy;
1057	n/a	}
1058	n/a	else {
1059	n/a	PyObject *w;
1060	n/a
1061	n/a	w = (PyObject*)_PyUnicode_New(length);
1062	n/a	if (w == NULL)
1063	n/a	return NULL;
1064	n/a	copy_length = _PyUnicode_WSTR_LENGTH(unicode);
1065	n/a	copy_length = Py_MIN(copy_length, length);
1066	n/a	memcpy(_PyUnicode_WSTR(w), _PyUnicode_WSTR(unicode),
1067	n/a	copy_length * sizeof(wchar_t));
1068	n/a	return w;
1069	n/a	}
1070	n/a	}
1071	n/a
1072	n/a	/* We allocate one more byte to make sure the string is
1073	n/a	Ux0000 terminated; some code (e.g. new_identifier)
1074	n/a	relies on that.
1075	n/a
1076	n/a	XXX This allocator could further be enhanced by assuring that the
1077	n/a	free list never reduces its size below 1.
1078	n/a
1079	n/a	*/
1080	n/a
1081	n/a	static PyUnicodeObject *
1082	n/a	_PyUnicode_New(Py_ssize_t length)
1083	n/a	{
1084	n/a	PyUnicodeObject *unicode;
1085	n/a	size_t new_size;
1086	n/a
1087	n/a	/* Optimization for empty strings */
1088	n/a	if (length == 0 && unicode_empty != NULL) {
1089	n/a	Py_INCREF(unicode_empty);
1090	n/a	return (PyUnicodeObject*)unicode_empty;
1091	n/a	}
1092	n/a
1093	n/a	/* Ensure we won't overflow the size. */
1094	n/a	if (length > ((PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(Py_UNICODE)) - 1)) {
1095	n/a	return (PyUnicodeObject *)PyErr_NoMemory();
1096	n/a	}
1097	n/a	if (length < 0) {
1098	n/a	PyErr_SetString(PyExc_SystemError,
1099	n/a	"Negative size passed to _PyUnicode_New");
1100	n/a	return NULL;
1101	n/a	}
1102	n/a
1103	n/a	unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type);
1104	n/a	if (unicode == NULL)
1105	n/a	return NULL;
1106	n/a	new_size = sizeof(Py_UNICODE) * ((size_t)length + 1);
1107	n/a
1108	n/a	_PyUnicode_WSTR_LENGTH(unicode) = length;
1109	n/a	_PyUnicode_HASH(unicode) = -1;
1110	n/a	_PyUnicode_STATE(unicode).interned = 0;
1111	n/a	_PyUnicode_STATE(unicode).kind = 0;
1112	n/a	_PyUnicode_STATE(unicode).compact = 0;
1113	n/a	_PyUnicode_STATE(unicode).ready = 0;
1114	n/a	_PyUnicode_STATE(unicode).ascii = 0;
1115	n/a	_PyUnicode_DATA_ANY(unicode) = NULL;
1116	n/a	_PyUnicode_LENGTH(unicode) = 0;
1117	n/a	_PyUnicode_UTF8(unicode) = NULL;
1118	n/a	_PyUnicode_UTF8_LENGTH(unicode) = 0;
1119	n/a
1120	n/a	_PyUnicode_WSTR(unicode) = (Py_UNICODE*) PyObject_MALLOC(new_size);
1121	n/a	if (!_PyUnicode_WSTR(unicode)) {
1122	n/a	Py_DECREF(unicode);
1123	n/a	PyErr_NoMemory();
1124	n/a	return NULL;
1125	n/a	}
1126	n/a
1127	n/a	/* Initialize the first element to guard against cases where
1128	n/a	* the caller fails before initializing str -- unicode_resize()
1129	n/a	* reads str[0], and the Keep-Alive optimization can keep memory
1130	n/a	* allocated for str alive across a call to unicode_dealloc(unicode).
1131	n/a	* We don't want unicode_resize to read uninitialized memory in
1132	n/a	* that case.
1133	n/a	*/
1134	n/a	_PyUnicode_WSTR(unicode)[0] = 0;
1135	n/a	_PyUnicode_WSTR(unicode)[length] = 0;
1136	n/a
1137	n/a	assert(_PyUnicode_CheckConsistency((PyObject *)unicode, 0));
1138	n/a	return unicode;
1139	n/a	}
1140	n/a
1141	n/a	static const char*
1142	n/a	unicode_kind_name(PyObject *unicode)
1143	n/a	{
1144	n/a	/* don't check consistency: unicode_kind_name() is called from
1145	n/a	_PyUnicode_Dump() */
1146	n/a	if (!PyUnicode_IS_COMPACT(unicode))
1147	n/a	{
1148	n/a	if (!PyUnicode_IS_READY(unicode))
1149	n/a	return "wstr";
1150	n/a	switch (PyUnicode_KIND(unicode))
1151	n/a	{
1152	n/a	case PyUnicode_1BYTE_KIND:
1153	n/a	if (PyUnicode_IS_ASCII(unicode))
1154	n/a	return "legacy ascii";
1155	n/a	else
1156	n/a	return "legacy latin1";
1157	n/a	case PyUnicode_2BYTE_KIND:
1158	n/a	return "legacy UCS2";
1159	n/a	case PyUnicode_4BYTE_KIND:
1160	n/a	return "legacy UCS4";
1161	n/a	default:
1162	n/a	return "<legacy invalid kind>";
1163	n/a	}
1164	n/a	}
1165	n/a	assert(PyUnicode_IS_READY(unicode));
1166	n/a	switch (PyUnicode_KIND(unicode)) {
1167	n/a	case PyUnicode_1BYTE_KIND:
1168	n/a	if (PyUnicode_IS_ASCII(unicode))
1169	n/a	return "ascii";
1170	n/a	else
1171	n/a	return "latin1";
1172	n/a	case PyUnicode_2BYTE_KIND:
1173	n/a	return "UCS2";
1174	n/a	case PyUnicode_4BYTE_KIND:
1175	n/a	return "UCS4";
1176	n/a	default:
1177	n/a	return "<invalid compact kind>";
1178	n/a	}
1179	n/a	}
1180	n/a
1181	n/a	#ifdef Py_DEBUG
1182	n/a	/* Functions wrapping macros for use in debugger */
1183	n/a	char _PyUnicode_utf8(void unicode){
1184	n/a	return PyUnicode_UTF8(unicode);
1185	n/a	}
1186	n/a
1187	n/a	void _PyUnicode_compact_data(void unicode) {
1188	n/a	return _PyUnicode_COMPACT_DATA(unicode);
1189	n/a	}
1190	n/a	void _PyUnicode_data(void unicode){
1191	n/a	printf("obj %p\n", unicode);
1192	n/a	printf("compact %d\n", PyUnicode_IS_COMPACT(unicode));
1193	n/a	printf("compact ascii %d\n", PyUnicode_IS_COMPACT_ASCII(unicode));
1194	n/a	printf("ascii op %p\n", ((void)((PyASCIIObject)(unicode) + 1)));
1195	n/a	printf("compact op %p\n", ((void)((PyCompactUnicodeObject)(unicode) + 1)));
1196	n/a	printf("compact data %p\n", _PyUnicode_COMPACT_DATA(unicode));
1197	n/a	return PyUnicode_DATA(unicode);
1198	n/a	}
1199	n/a
1200	n/a	void
1201	n/a	_PyUnicode_Dump(PyObject *op)
1202	n/a	{
1203	n/a	PyASCIIObject ascii = (PyASCIIObject )op;
1204	n/a	PyCompactUnicodeObject compact = (PyCompactUnicodeObject )op;
1205	n/a	PyUnicodeObject unicode = (PyUnicodeObject )op;
1206	n/a	void *data;
1207	n/a
1208	n/a	if (ascii->state.compact)
1209	n/a	{
1210	n/a	if (ascii->state.ascii)
1211	n/a	data = (ascii + 1);
1212	n/a	else
1213	n/a	data = (compact + 1);
1214	n/a	}
1215	n/a	else
1216	n/a	data = unicode->data.any;
1217	n/a	printf("%s: len=%" PY_FORMAT_SIZE_T "u, ",
1218	n/a	unicode_kind_name(op), ascii->length);
1219	n/a
1220	n/a	if (ascii->wstr == data)
1221	n/a	printf("shared ");
1222	n/a	printf("wstr=%p", ascii->wstr);
1223	n/a
1224	n/a	if (!(ascii->state.ascii == 1 && ascii->state.compact == 1)) {
1225	n/a	printf(" (%" PY_FORMAT_SIZE_T "u), ", compact->wstr_length);
1226	n/a	if (!ascii->state.compact && compact->utf8 == unicode->data.any)
1227	n/a	printf("shared ");
1228	n/a	printf("utf8=%p (%" PY_FORMAT_SIZE_T "u)",
1229	n/a	compact->utf8, compact->utf8_length);
1230	n/a	}
1231	n/a	printf(", data=%p\n", data);
1232	n/a	}
1233	n/a	#endif
1234	n/a
1235	n/a	PyObject *
1236	n/a	PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar)
1237	n/a	{
1238	n/a	PyObject *obj;
1239	n/a	PyCompactUnicodeObject *unicode;
1240	n/a	void *data;
1241	n/a	enum PyUnicode_Kind kind;
1242	n/a	int is_sharing, is_ascii;
1243	n/a	Py_ssize_t char_size;
1244	n/a	Py_ssize_t struct_size;
1245	n/a
1246	n/a	/* Optimization for empty strings */
1247	n/a	if (size == 0 && unicode_empty != NULL) {
1248	n/a	Py_INCREF(unicode_empty);
1249	n/a	return unicode_empty;
1250	n/a	}
1251	n/a
1252	n/a	is_ascii = 0;
1253	n/a	is_sharing = 0;
1254	n/a	struct_size = sizeof(PyCompactUnicodeObject);
1255	n/a	if (maxchar < 128) {
1256	n/a	kind = PyUnicode_1BYTE_KIND;
1257	n/a	char_size = 1;
1258	n/a	is_ascii = 1;
1259	n/a	struct_size = sizeof(PyASCIIObject);
1260	n/a	}
1261	n/a	else if (maxchar < 256) {
1262	n/a	kind = PyUnicode_1BYTE_KIND;
1263	n/a	char_size = 1;
1264	n/a	}
1265	n/a	else if (maxchar < 65536) {
1266	n/a	kind = PyUnicode_2BYTE_KIND;
1267	n/a	char_size = 2;
1268	n/a	if (sizeof(wchar_t) == 2)
1269	n/a	is_sharing = 1;
1270	n/a	}
1271	n/a	else {
1272	n/a	if (maxchar > MAX_UNICODE) {
1273	n/a	PyErr_SetString(PyExc_SystemError,
1274	n/a	"invalid maximum character passed to PyUnicode_New");
1275	n/a	return NULL;
1276	n/a	}
1277	n/a	kind = PyUnicode_4BYTE_KIND;
1278	n/a	char_size = 4;
1279	n/a	if (sizeof(wchar_t) == 4)
1280	n/a	is_sharing = 1;
1281	n/a	}
1282	n/a
1283	n/a	/* Ensure we won't overflow the size. */
1284	n/a	if (size < 0) {
1285	n/a	PyErr_SetString(PyExc_SystemError,
1286	n/a	"Negative size passed to PyUnicode_New");
1287	n/a	return NULL;
1288	n/a	}
1289	n/a	if (size > ((PY_SSIZE_T_MAX - struct_size) / char_size - 1))
1290	n/a	return PyErr_NoMemory();
1291	n/a
1292	n/a	/* Duplicated allocation code from _PyObject_New() instead of a call to
1293	n/a	* PyObject_New() so we are able to allocate space for the object and
1294	n/a	* it's data buffer.
1295	n/a	*/
1296	n/a	obj = (PyObject ) PyObject_MALLOC(struct_size + (size + 1) char_size);
1297	n/a	if (obj == NULL)
1298	n/a	return PyErr_NoMemory();
1299	n/a	obj = PyObject_INIT(obj, &PyUnicode_Type);
1300	n/a	if (obj == NULL)
1301	n/a	return NULL;
1302	n/a
1303	n/a	unicode = (PyCompactUnicodeObject *)obj;
1304	n/a	if (is_ascii)
1305	n/a	data = ((PyASCIIObject*)obj) + 1;
1306	n/a	else
1307	n/a	data = unicode + 1;
1308	n/a	_PyUnicode_LENGTH(unicode) = size;
1309	n/a	_PyUnicode_HASH(unicode) = -1;
1310	n/a	_PyUnicode_STATE(unicode).interned = 0;
1311	n/a	_PyUnicode_STATE(unicode).kind = kind;
1312	n/a	_PyUnicode_STATE(unicode).compact = 1;
1313	n/a	_PyUnicode_STATE(unicode).ready = 1;
1314	n/a	_PyUnicode_STATE(unicode).ascii = is_ascii;
1315	n/a	if (is_ascii) {
1316	n/a	((char*)data)[size] = 0;
1317	n/a	_PyUnicode_WSTR(unicode) = NULL;
1318	n/a	}
1319	n/a	else if (kind == PyUnicode_1BYTE_KIND) {
1320	n/a	((char*)data)[size] = 0;
1321	n/a	_PyUnicode_WSTR(unicode) = NULL;
1322	n/a	_PyUnicode_WSTR_LENGTH(unicode) = 0;
1323	n/a	unicode->utf8 = NULL;
1324	n/a	unicode->utf8_length = 0;
1325	n/a	}
1326	n/a	else {
1327	n/a	unicode->utf8 = NULL;
1328	n/a	unicode->utf8_length = 0;
1329	n/a	if (kind == PyUnicode_2BYTE_KIND)
1330	n/a	((Py_UCS2*)data)[size] = 0;
1331	n/a	else /* kind == PyUnicode_4BYTE_KIND */
1332	n/a	((Py_UCS4*)data)[size] = 0;
1333	n/a	if (is_sharing) {
1334	n/a	_PyUnicode_WSTR_LENGTH(unicode) = size;
1335	n/a	_PyUnicode_WSTR(unicode) = (wchar_t *)data;
1336	n/a	}
1337	n/a	else {
1338	n/a	_PyUnicode_WSTR_LENGTH(unicode) = 0;
1339	n/a	_PyUnicode_WSTR(unicode) = NULL;
1340	n/a	}
1341	n/a	}
1342	n/a	#ifdef Py_DEBUG
1343	n/a	unicode_fill_invalid((PyObject*)unicode, 0);
1344	n/a	#endif
1345	n/a	assert(_PyUnicode_CheckConsistency((PyObject*)unicode, 0));
1346	n/a	return obj;
1347	n/a	}
1348	n/a
1349	n/a	#if SIZEOF_WCHAR_T == 2
1350	n/a	/* Helper function to convert a 16-bits wchar_t representation to UCS4, this
1351	n/a	will decode surrogate pairs, the other conversions are implemented as macros
1352	n/a	for efficiency.
1353	n/a
1354	n/a	This function assumes that unicode can hold one more code point than wstr
1355	n/a	characters for a terminating null character. */
1356	n/a	static void
1357	n/a	unicode_convert_wchar_to_ucs4(const wchar_t begin, const wchar_t end,
1358	n/a	PyObject *unicode)
1359	n/a	{
1360	n/a	const wchar_t *iter;
1361	n/a	Py_UCS4 *ucs4_out;
1362	n/a
1363	n/a	assert(unicode != NULL);
1364	n/a	assert(_PyUnicode_CHECK(unicode));
1365	n/a	assert(_PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND);
1366	n/a	ucs4_out = PyUnicode_4BYTE_DATA(unicode);
1367	n/a
1368	n/a	for (iter = begin; iter < end; ) {
1369	n/a	assert(ucs4_out < (PyUnicode_4BYTE_DATA(unicode) +
1370	n/a	_PyUnicode_GET_LENGTH(unicode)));
1371	n/a	if (Py_UNICODE_IS_HIGH_SURROGATE(iter[0])
1372	n/a	&& (iter+1) < end
1373	n/a	&& Py_UNICODE_IS_LOW_SURROGATE(iter[1]))
1374	n/a	{
1375	n/a	*ucs4_out++ = Py_UNICODE_JOIN_SURROGATES(iter[0], iter[1]);
1376	n/a	iter += 2;
1377	n/a	}
1378	n/a	else {
1379	n/a	ucs4_out++ = iter;
1380	n/a	iter++;
1381	n/a	}
1382	n/a	}
1383	n/a	assert(ucs4_out == (PyUnicode_4BYTE_DATA(unicode) +
1384	n/a	_PyUnicode_GET_LENGTH(unicode)));
1385	n/a
1386	n/a	}
1387	n/a	#endif
1388	n/a
1389	n/a	static int
1390	n/a	unicode_check_modifiable(PyObject *unicode)
1391	n/a	{
1392	n/a	if (!unicode_modifiable(unicode)) {
1393	n/a	PyErr_SetString(PyExc_SystemError,
1394	n/a	"Cannot modify a string currently used");
1395	n/a	return -1;
1396	n/a	}
1397	n/a	return 0;
1398	n/a	}
1399	n/a
1400	n/a	static int
1401	n/a	_copy_characters(PyObject *to, Py_ssize_t to_start,
1402	n/a	PyObject *from, Py_ssize_t from_start,
1403	n/a	Py_ssize_t how_many, int check_maxchar)
1404	n/a	{
1405	n/a	unsigned int from_kind, to_kind;
1406	n/a	void from_data, to_data;
1407	n/a
1408	n/a	assert(0 <= how_many);
1409	n/a	assert(0 <= from_start);
1410	n/a	assert(0 <= to_start);
1411	n/a	assert(PyUnicode_Check(from));
1412	n/a	assert(PyUnicode_IS_READY(from));
1413	n/a	assert(from_start + how_many <= PyUnicode_GET_LENGTH(from));
1414	n/a
1415	n/a	assert(PyUnicode_Check(to));
1416	n/a	assert(PyUnicode_IS_READY(to));
1417	n/a	assert(to_start + how_many <= PyUnicode_GET_LENGTH(to));
1418	n/a
1419	n/a	if (how_many == 0)
1420	n/a	return 0;
1421	n/a
1422	n/a	from_kind = PyUnicode_KIND(from);
1423	n/a	from_data = PyUnicode_DATA(from);
1424	n/a	to_kind = PyUnicode_KIND(to);
1425	n/a	to_data = PyUnicode_DATA(to);
1426	n/a
1427	n/a	#ifdef Py_DEBUG
1428	n/a	if (!check_maxchar
1429	n/a	&& PyUnicode_MAX_CHAR_VALUE(from) > PyUnicode_MAX_CHAR_VALUE(to))
1430	n/a	{
1431	n/a	const Py_UCS4 to_maxchar = PyUnicode_MAX_CHAR_VALUE(to);
1432	n/a	Py_UCS4 ch;
1433	n/a	Py_ssize_t i;
1434	n/a	for (i=0; i < how_many; i++) {
1435	n/a	ch = PyUnicode_READ(from_kind, from_data, from_start + i);
1436	n/a	assert(ch <= to_maxchar);
1437	n/a	}
1438	n/a	}
1439	n/a	#endif
1440	n/a
1441	n/a	if (from_kind == to_kind) {
1442	n/a	if (check_maxchar
1443	n/a	&& !PyUnicode_IS_ASCII(from) && PyUnicode_IS_ASCII(to))
1444	n/a	{
1445	n/a	/* Writing Latin-1 characters into an ASCII string requires to
1446	n/a	check that all written characters are pure ASCII */
1447	n/a	Py_UCS4 max_char;
1448	n/a	max_char = ucs1lib_find_max_char(from_data,
1449	n/a	(Py_UCS1*)from_data + how_many);
1450	n/a	if (max_char >= 128)
1451	n/a	return -1;
1452	n/a	}
1453	n/a	memcpy((char)to_data + to_kind to_start,
1454	n/a	(char)from_data + from_kind from_start,
1455	n/a	to_kind * how_many);
1456	n/a	}
1457	n/a	else if (from_kind == PyUnicode_1BYTE_KIND
1458	n/a	&& to_kind == PyUnicode_2BYTE_KIND)
1459	n/a	{
1460	n/a	_PyUnicode_CONVERT_BYTES(
1461	n/a	Py_UCS1, Py_UCS2,
1462	n/a	PyUnicode_1BYTE_DATA(from) + from_start,
1463	n/a	PyUnicode_1BYTE_DATA(from) + from_start + how_many,
1464	n/a	PyUnicode_2BYTE_DATA(to) + to_start
1465	n/a	);
1466	n/a	}
1467	n/a	else if (from_kind == PyUnicode_1BYTE_KIND
1468	n/a	&& to_kind == PyUnicode_4BYTE_KIND)
1469	n/a	{
1470	n/a	_PyUnicode_CONVERT_BYTES(
1471	n/a	Py_UCS1, Py_UCS4,
1472	n/a	PyUnicode_1BYTE_DATA(from) + from_start,
1473	n/a	PyUnicode_1BYTE_DATA(from) + from_start + how_many,
1474	n/a	PyUnicode_4BYTE_DATA(to) + to_start
1475	n/a	);
1476	n/a	}
1477	n/a	else if (from_kind == PyUnicode_2BYTE_KIND
1478	n/a	&& to_kind == PyUnicode_4BYTE_KIND)
1479	n/a	{
1480	n/a	_PyUnicode_CONVERT_BYTES(
1481	n/a	Py_UCS2, Py_UCS4,
1482	n/a	PyUnicode_2BYTE_DATA(from) + from_start,
1483	n/a	PyUnicode_2BYTE_DATA(from) + from_start + how_many,
1484	n/a	PyUnicode_4BYTE_DATA(to) + to_start
1485	n/a	);
1486	n/a	}
1487	n/a	else {
1488	n/a	assert (PyUnicode_MAX_CHAR_VALUE(from) > PyUnicode_MAX_CHAR_VALUE(to));
1489	n/a
1490	n/a	if (!check_maxchar) {
1491	n/a	if (from_kind == PyUnicode_2BYTE_KIND
1492	n/a	&& to_kind == PyUnicode_1BYTE_KIND)
1493	n/a	{
1494	n/a	_PyUnicode_CONVERT_BYTES(
1495	n/a	Py_UCS2, Py_UCS1,
1496	n/a	PyUnicode_2BYTE_DATA(from) + from_start,
1497	n/a	PyUnicode_2BYTE_DATA(from) + from_start + how_many,
1498	n/a	PyUnicode_1BYTE_DATA(to) + to_start
1499	n/a	);
1500	n/a	}
1501	n/a	else if (from_kind == PyUnicode_4BYTE_KIND
1502	n/a	&& to_kind == PyUnicode_1BYTE_KIND)
1503	n/a	{
1504	n/a	_PyUnicode_CONVERT_BYTES(
1505	n/a	Py_UCS4, Py_UCS1,
1506	n/a	PyUnicode_4BYTE_DATA(from) + from_start,
1507	n/a	PyUnicode_4BYTE_DATA(from) + from_start + how_many,
1508	n/a	PyUnicode_1BYTE_DATA(to) + to_start
1509	n/a	);
1510	n/a	}
1511	n/a	else if (from_kind == PyUnicode_4BYTE_KIND
1512	n/a	&& to_kind == PyUnicode_2BYTE_KIND)
1513	n/a	{
1514	n/a	_PyUnicode_CONVERT_BYTES(
1515	n/a	Py_UCS4, Py_UCS2,
1516	n/a	PyUnicode_4BYTE_DATA(from) + from_start,
1517	n/a	PyUnicode_4BYTE_DATA(from) + from_start + how_many,
1518	n/a	PyUnicode_2BYTE_DATA(to) + to_start
1519	n/a	);
1520	n/a	}
1521	n/a	else {
1522	n/a	assert(0);
1523	n/a	return -1;
1524	n/a	}
1525	n/a	}
1526	n/a	else {
1527	n/a	const Py_UCS4 to_maxchar = PyUnicode_MAX_CHAR_VALUE(to);
1528	n/a	Py_UCS4 ch;
1529	n/a	Py_ssize_t i;
1530	n/a
1531	n/a	for (i=0; i < how_many; i++) {
1532	n/a	ch = PyUnicode_READ(from_kind, from_data, from_start + i);
1533	n/a	if (ch > to_maxchar)
1534	n/a	return -1;
1535	n/a	PyUnicode_WRITE(to_kind, to_data, to_start + i, ch);
1536	n/a	}
1537	n/a	}
1538	n/a	}
1539	n/a	return 0;
1540	n/a	}
1541	n/a
1542	n/a	void
1543	n/a	_PyUnicode_FastCopyCharacters(
1544	n/a	PyObject *to, Py_ssize_t to_start,
1545	n/a	PyObject *from, Py_ssize_t from_start, Py_ssize_t how_many)
1546	n/a	{
1547	n/a	(void)_copy_characters(to, to_start, from, from_start, how_many, 0);
1548	n/a	}
1549	n/a
1550	n/a	Py_ssize_t
1551	n/a	PyUnicode_CopyCharacters(PyObject *to, Py_ssize_t to_start,
1552	n/a	PyObject *from, Py_ssize_t from_start,
1553	n/a	Py_ssize_t how_many)
1554	n/a	{
1555	n/a	int err;
1556	n/a
1557	n/a	if (!PyUnicode_Check(from) \|\| !PyUnicode_Check(to)) {
1558	n/a	PyErr_BadInternalCall();
1559	n/a	return -1;
1560	n/a	}
1561	n/a
1562	n/a	if (PyUnicode_READY(from) == -1)
1563	n/a	return -1;
1564	n/a	if (PyUnicode_READY(to) == -1)
1565	n/a	return -1;
1566	n/a
1567	n/a	if ((size_t)from_start > (size_t)PyUnicode_GET_LENGTH(from)) {
1568	n/a	PyErr_SetString(PyExc_IndexError, "string index out of range");
1569	n/a	return -1;
1570	n/a	}
1571	n/a	if ((size_t)to_start > (size_t)PyUnicode_GET_LENGTH(to)) {
1572	n/a	PyErr_SetString(PyExc_IndexError, "string index out of range");
1573	n/a	return -1;
1574	n/a	}
1575	n/a	if (how_many < 0) {
1576	n/a	PyErr_SetString(PyExc_SystemError, "how_many cannot be negative");
1577	n/a	return -1;
1578	n/a	}
1579	n/a	how_many = Py_MIN(PyUnicode_GET_LENGTH(from)-from_start, how_many);
1580	n/a	if (to_start + how_many > PyUnicode_GET_LENGTH(to)) {
1581	n/a	PyErr_Format(PyExc_SystemError,
1582	n/a	"Cannot write %zi characters at %zi "
1583	n/a	"in a string of %zi characters",
1584	n/a	how_many, to_start, PyUnicode_GET_LENGTH(to));
1585	n/a	return -1;
1586	n/a	}
1587	n/a
1588	n/a	if (how_many == 0)
1589	n/a	return 0;
1590	n/a
1591	n/a	if (unicode_check_modifiable(to))
1592	n/a	return -1;
1593	n/a
1594	n/a	err = _copy_characters(to, to_start, from, from_start, how_many, 1);
1595	n/a	if (err) {
1596	n/a	PyErr_Format(PyExc_SystemError,
1597	n/a	"Cannot copy %s characters "
1598	n/a	"into a string of %s characters",
1599	n/a	unicode_kind_name(from),
1600	n/a	unicode_kind_name(to));
1601	n/a	return -1;
1602	n/a	}
1603	n/a	return how_many;
1604	n/a	}
1605	n/a
1606	n/a	/* Find the maximum code point and count the number of surrogate pairs so a
1607	n/a	correct string length can be computed before converting a string to UCS4.
1608	n/a	This function counts single surrogates as a character and not as a pair.
1609	n/a
1610	n/a	Return 0 on success, or -1 on error. */
1611	n/a	static int
1612	n/a	find_maxchar_surrogates(const wchar_t begin, const wchar_t end,
1613	n/a	Py_UCS4 maxchar, Py_ssize_t num_surrogates)
1614	n/a	{
1615	n/a	const wchar_t *iter;
1616	n/a	Py_UCS4 ch;
1617	n/a
1618	n/a	assert(num_surrogates != NULL && maxchar != NULL);
1619	n/a	*num_surrogates = 0;
1620	n/a	*maxchar = 0;
1621	n/a
1622	n/a	for (iter = begin; iter < end; ) {
1623	n/a	#if SIZEOF_WCHAR_T == 2
1624	n/a	if (Py_UNICODE_IS_HIGH_SURROGATE(iter[0])
1625	n/a	&& (iter+1) < end
1626	n/a	&& Py_UNICODE_IS_LOW_SURROGATE(iter[1]))
1627	n/a	{
1628	n/a	ch = Py_UNICODE_JOIN_SURROGATES(iter[0], iter[1]);
1629	n/a	++(*num_surrogates);
1630	n/a	iter += 2;
1631	n/a	}
1632	n/a	else
1633	n/a	#endif
1634	n/a	{
1635	n/a	ch = *iter;
1636	n/a	iter++;
1637	n/a	}
1638	n/a	if (ch > *maxchar) {
1639	n/a	*maxchar = ch;
1640	n/a	if (*maxchar > MAX_UNICODE) {
1641	n/a	PyErr_Format(PyExc_ValueError,
1642	n/a	"character U+%x is not in range [U+0000; U+10ffff]",
1643	n/a	ch);
1644	n/a	return -1;
1645	n/a	}
1646	n/a	}
1647	n/a	}
1648	n/a	return 0;
1649	n/a	}
1650	n/a
1651	n/a	int
1652	n/a	_PyUnicode_Ready(PyObject *unicode)
1653	n/a	{
1654	n/a	wchar_t *end;
1655	n/a	Py_UCS4 maxchar = 0;
1656	n/a	Py_ssize_t num_surrogates;
1657	n/a	#if SIZEOF_WCHAR_T == 2
1658	n/a	Py_ssize_t length_wo_surrogates;
1659	n/a	#endif
1660	n/a
1661	n/a	/* _PyUnicode_Ready() is only intended for old-style API usage where
1662	n/a	strings were created using _PyObject_New() and where no canonical
1663	n/a	representation (the str field) has been set yet aka strings
1664	n/a	which are not yet ready. */
1665	n/a	assert(_PyUnicode_CHECK(unicode));
1666	n/a	assert(_PyUnicode_KIND(unicode) == PyUnicode_WCHAR_KIND);
1667	n/a	assert(_PyUnicode_WSTR(unicode) != NULL);
1668	n/a	assert(_PyUnicode_DATA_ANY(unicode) == NULL);
1669	n/a	assert(_PyUnicode_UTF8(unicode) == NULL);
1670	n/a	/* Actually, it should neither be interned nor be anything else: */
1671	n/a	assert(_PyUnicode_STATE(unicode).interned == SSTATE_NOT_INTERNED);
1672	n/a
1673	n/a	end = _PyUnicode_WSTR(unicode) + _PyUnicode_WSTR_LENGTH(unicode);
1674	n/a	if (find_maxchar_surrogates(_PyUnicode_WSTR(unicode), end,
1675	n/a	&maxchar, &num_surrogates) == -1)
1676	n/a	return -1;
1677	n/a
1678	n/a	if (maxchar < 256) {
1679	n/a	_PyUnicode_DATA_ANY(unicode) = PyObject_MALLOC(_PyUnicode_WSTR_LENGTH(unicode) + 1);
1680	n/a	if (!_PyUnicode_DATA_ANY(unicode)) {
1681	n/a	PyErr_NoMemory();
1682	n/a	return -1;
1683	n/a	}
1684	n/a	_PyUnicode_CONVERT_BYTES(wchar_t, unsigned char,
1685	n/a	_PyUnicode_WSTR(unicode), end,
1686	n/a	PyUnicode_1BYTE_DATA(unicode));
1687	n/a	PyUnicode_1BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0';
1688	n/a	_PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode);
1689	n/a	_PyUnicode_STATE(unicode).kind = PyUnicode_1BYTE_KIND;
1690	n/a	if (maxchar < 128) {
1691	n/a	_PyUnicode_STATE(unicode).ascii = 1;
1692	n/a	_PyUnicode_UTF8(unicode) = _PyUnicode_DATA_ANY(unicode);
1693	n/a	_PyUnicode_UTF8_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode);
1694	n/a	}
1695	n/a	else {
1696	n/a	_PyUnicode_STATE(unicode).ascii = 0;
1697	n/a	_PyUnicode_UTF8(unicode) = NULL;
1698	n/a	_PyUnicode_UTF8_LENGTH(unicode) = 0;
1699	n/a	}
1700	n/a	PyObject_FREE(_PyUnicode_WSTR(unicode));
1701	n/a	_PyUnicode_WSTR(unicode) = NULL;
1702	n/a	_PyUnicode_WSTR_LENGTH(unicode) = 0;
1703	n/a	}
1704	n/a	/* In this case we might have to convert down from 4-byte native
1705	n/a	wchar_t to 2-byte unicode. */
1706	n/a	else if (maxchar < 65536) {
1707	n/a	assert(num_surrogates == 0 &&
1708	n/a	"FindMaxCharAndNumSurrogatePairs() messed up");
1709	n/a
1710	n/a	#if SIZEOF_WCHAR_T == 2
1711	n/a	/* We can share representations and are done. */
1712	n/a	_PyUnicode_DATA_ANY(unicode) = _PyUnicode_WSTR(unicode);
1713	n/a	PyUnicode_2BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0';
1714	n/a	_PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode);
1715	n/a	_PyUnicode_STATE(unicode).kind = PyUnicode_2BYTE_KIND;
1716	n/a	_PyUnicode_UTF8(unicode) = NULL;
1717	n/a	_PyUnicode_UTF8_LENGTH(unicode) = 0;
1718	n/a	#else
1719	n/a	/* sizeof(wchar_t) == 4 */
1720	n/a	_PyUnicode_DATA_ANY(unicode) = PyObject_MALLOC(
1721	n/a	2 * (_PyUnicode_WSTR_LENGTH(unicode) + 1));
1722	n/a	if (!_PyUnicode_DATA_ANY(unicode)) {
1723	n/a	PyErr_NoMemory();
1724	n/a	return -1;
1725	n/a	}
1726	n/a	_PyUnicode_CONVERT_BYTES(wchar_t, Py_UCS2,
1727	n/a	_PyUnicode_WSTR(unicode), end,
1728	n/a	PyUnicode_2BYTE_DATA(unicode));
1729	n/a	PyUnicode_2BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0';
1730	n/a	_PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode);
1731	n/a	_PyUnicode_STATE(unicode).kind = PyUnicode_2BYTE_KIND;
1732	n/a	_PyUnicode_UTF8(unicode) = NULL;
1733	n/a	_PyUnicode_UTF8_LENGTH(unicode) = 0;
1734	n/a	PyObject_FREE(_PyUnicode_WSTR(unicode));
1735	n/a	_PyUnicode_WSTR(unicode) = NULL;
1736	n/a	_PyUnicode_WSTR_LENGTH(unicode) = 0;
1737	n/a	#endif
1738	n/a	}
1739	n/a	/* maxchar exeeds 16 bit, wee need 4 bytes for unicode characters */
1740	n/a	else {
1741	n/a	#if SIZEOF_WCHAR_T == 2
1742	n/a	/* in case the native representation is 2-bytes, we need to allocate a
1743	n/a	new normalized 4-byte version. */
1744	n/a	length_wo_surrogates = _PyUnicode_WSTR_LENGTH(unicode) - num_surrogates;
1745	n/a	if (length_wo_surrogates > PY_SSIZE_T_MAX / 4 - 1) {
1746	n/a	PyErr_NoMemory();
1747	n/a	return -1;
1748	n/a	}
1749	n/a	_PyUnicode_DATA_ANY(unicode) = PyObject_MALLOC(4 * (length_wo_surrogates + 1));
1750	n/a	if (!_PyUnicode_DATA_ANY(unicode)) {
1751	n/a	PyErr_NoMemory();
1752	n/a	return -1;
1753	n/a	}
1754	n/a	_PyUnicode_LENGTH(unicode) = length_wo_surrogates;
1755	n/a	_PyUnicode_STATE(unicode).kind = PyUnicode_4BYTE_KIND;
1756	n/a	_PyUnicode_UTF8(unicode) = NULL;
1757	n/a	_PyUnicode_UTF8_LENGTH(unicode) = 0;
1758	n/a	/* unicode_convert_wchar_to_ucs4() requires a ready string */
1759	n/a	_PyUnicode_STATE(unicode).ready = 1;
1760	n/a	unicode_convert_wchar_to_ucs4(_PyUnicode_WSTR(unicode), end, unicode);
1761	n/a	PyObject_FREE(_PyUnicode_WSTR(unicode));
1762	n/a	_PyUnicode_WSTR(unicode) = NULL;
1763	n/a	_PyUnicode_WSTR_LENGTH(unicode) = 0;
1764	n/a	#else
1765	n/a	assert(num_surrogates == 0);
1766	n/a
1767	n/a	_PyUnicode_DATA_ANY(unicode) = _PyUnicode_WSTR(unicode);
1768	n/a	_PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode);
1769	n/a	_PyUnicode_UTF8(unicode) = NULL;
1770	n/a	_PyUnicode_UTF8_LENGTH(unicode) = 0;
1771	n/a	_PyUnicode_STATE(unicode).kind = PyUnicode_4BYTE_KIND;
1772	n/a	#endif
1773	n/a	PyUnicode_4BYTE_DATA(unicode)[_PyUnicode_LENGTH(unicode)] = '\0';
1774	n/a	}
1775	n/a	_PyUnicode_STATE(unicode).ready = 1;
1776	n/a	assert(_PyUnicode_CheckConsistency(unicode, 1));
1777	n/a	return 0;
1778	n/a	}
1779	n/a
1780	n/a	static void
1781	n/a	unicode_dealloc(PyObject *unicode)
1782	n/a	{
1783	n/a	switch (PyUnicode_CHECK_INTERNED(unicode)) {
1784	n/a	case SSTATE_NOT_INTERNED:
1785	n/a	break;
1786	n/a
1787	n/a	case SSTATE_INTERNED_MORTAL:
1788	n/a	/* revive dead object temporarily for DelItem */
1789	n/a	Py_REFCNT(unicode) = 3;
1790	n/a	if (PyDict_DelItem(interned, unicode) != 0)
1791	n/a	Py_FatalError(
1792	n/a	"deletion of interned string failed");
1793	n/a	break;
1794	n/a
1795	n/a	case SSTATE_INTERNED_IMMORTAL:
1796	n/a	Py_FatalError("Immortal interned string died.");
1797	n/a
1798	n/a	default:
1799	n/a	Py_FatalError("Inconsistent interned string state.");
1800	n/a	}
1801	n/a
1802	n/a	if (_PyUnicode_HAS_WSTR_MEMORY(unicode))
1803	n/a	PyObject_DEL(_PyUnicode_WSTR(unicode));
1804	n/a	if (_PyUnicode_HAS_UTF8_MEMORY(unicode))
1805	n/a	PyObject_DEL(_PyUnicode_UTF8(unicode));
1806	n/a	if (!PyUnicode_IS_COMPACT(unicode) && _PyUnicode_DATA_ANY(unicode))
1807	n/a	PyObject_DEL(_PyUnicode_DATA_ANY(unicode));
1808	n/a
1809	n/a	Py_TYPE(unicode)->tp_free(unicode);
1810	n/a	}
1811	n/a
1812	n/a	#ifdef Py_DEBUG
1813	n/a	static int
1814	n/a	unicode_is_singleton(PyObject *unicode)
1815	n/a	{
1816	n/a	PyASCIIObject ascii = (PyASCIIObject )unicode;
1817	n/a	if (unicode == unicode_empty)
1818	n/a	return 1;
1819	n/a	if (ascii->state.kind != PyUnicode_WCHAR_KIND && ascii->length == 1)
1820	n/a	{
1821	n/a	Py_UCS4 ch = PyUnicode_READ_CHAR(unicode, 0);
1822	n/a	if (ch < 256 && unicode_latin1[ch] == unicode)
1823	n/a	return 1;
1824	n/a	}
1825	n/a	return 0;
1826	n/a	}
1827	n/a	#endif
1828	n/a
1829	n/a	static int
1830	n/a	unicode_modifiable(PyObject *unicode)
1831	n/a	{
1832	n/a	assert(_PyUnicode_CHECK(unicode));
1833	n/a	if (Py_REFCNT(unicode) != 1)
1834	n/a	return 0;
1835	n/a	if (_PyUnicode_HASH(unicode) != -1)
1836	n/a	return 0;
1837	n/a	if (PyUnicode_CHECK_INTERNED(unicode))
1838	n/a	return 0;
1839	n/a	if (!PyUnicode_CheckExact(unicode))
1840	n/a	return 0;
1841	n/a	#ifdef Py_DEBUG
1842	n/a	/* singleton refcount is greater than 1 */
1843	n/a	assert(!unicode_is_singleton(unicode));
1844	n/a	#endif
1845	n/a	return 1;
1846	n/a	}
1847	n/a
1848	n/a	static int
1849	n/a	unicode_resize(PyObject **p_unicode, Py_ssize_t length)
1850	n/a	{
1851	n/a	PyObject *unicode;
1852	n/a	Py_ssize_t old_length;
1853	n/a
1854	n/a	assert(p_unicode != NULL);
1855	n/a	unicode = *p_unicode;
1856	n/a
1857	n/a	assert(unicode != NULL);
1858	n/a	assert(PyUnicode_Check(unicode));
1859	n/a	assert(0 <= length);
1860	n/a
1861	n/a	if (_PyUnicode_KIND(unicode) == PyUnicode_WCHAR_KIND)
1862	n/a	old_length = PyUnicode_WSTR_LENGTH(unicode);
1863	n/a	else
1864	n/a	old_length = PyUnicode_GET_LENGTH(unicode);
1865	n/a	if (old_length == length)
1866	n/a	return 0;
1867	n/a
1868	n/a	if (length == 0) {
1869	n/a	_Py_INCREF_UNICODE_EMPTY();
1870	n/a	if (!unicode_empty)
1871	n/a	return -1;
1872	n/a	Py_SETREF(*p_unicode, unicode_empty);
1873	n/a	return 0;
1874	n/a	}
1875	n/a
1876	n/a	if (!unicode_modifiable(unicode)) {
1877	n/a	PyObject *copy = resize_copy(unicode, length);
1878	n/a	if (copy == NULL)
1879	n/a	return -1;
1880	n/a	Py_SETREF(*p_unicode, copy);
1881	n/a	return 0;
1882	n/a	}
1883	n/a
1884	n/a	if (PyUnicode_IS_COMPACT(unicode)) {
1885	n/a	PyObject *new_unicode = resize_compact(unicode, length);
1886	n/a	if (new_unicode == NULL)
1887	n/a	return -1;
1888	n/a	*p_unicode = new_unicode;
1889	n/a	return 0;
1890	n/a	}
1891	n/a	return resize_inplace(unicode, length);
1892	n/a	}
1893	n/a
1894	n/a	int
1895	n/a	PyUnicode_Resize(PyObject **p_unicode, Py_ssize_t length)
1896	n/a	{
1897	n/a	PyObject *unicode;
1898	n/a	if (p_unicode == NULL) {
1899	n/a	PyErr_BadInternalCall();
1900	n/a	return -1;
1901	n/a	}
1902	n/a	unicode = *p_unicode;
1903	n/a	if (unicode == NULL \|\| !PyUnicode_Check(unicode) \|\| length < 0)
1904	n/a	{
1905	n/a	PyErr_BadInternalCall();
1906	n/a	return -1;
1907	n/a	}
1908	n/a	return unicode_resize(p_unicode, length);
1909	n/a	}
1910	n/a
1911	n/a	/* Copy an ASCII or latin1 char* string into a Python Unicode string.
1912	n/a
1913	n/a	WARNING: The function doesn't copy the terminating null character and
1914	n/a	doesn't check the maximum character (may write a latin1 character in an
1915	n/a	ASCII string). */
1916	n/a	static void
1917	n/a	unicode_write_cstr(PyObject *unicode, Py_ssize_t index,
1918	n/a	const char *str, Py_ssize_t len)
1919	n/a	{
1920	n/a	enum PyUnicode_Kind kind = PyUnicode_KIND(unicode);
1921	n/a	void *data = PyUnicode_DATA(unicode);
1922	n/a	const char *end = str + len;
1923	n/a
1924	n/a	switch (kind) {
1925	n/a	case PyUnicode_1BYTE_KIND: {
1926	n/a	assert(index + len <= PyUnicode_GET_LENGTH(unicode));
1927	n/a	#ifdef Py_DEBUG
1928	n/a	if (PyUnicode_IS_ASCII(unicode)) {
1929	n/a	Py_UCS4 maxchar = ucs1lib_find_max_char(
1930	n/a	(const Py_UCS1*)str,
1931	n/a	(const Py_UCS1*)str + len);
1932	n/a	assert(maxchar < 128);
1933	n/a	}
1934	n/a	#endif
1935	n/a	memcpy((char *) data + index, str, len);
1936	n/a	break;
1937	n/a	}
1938	n/a	case PyUnicode_2BYTE_KIND: {
1939	n/a	Py_UCS2 start = (Py_UCS2 )data + index;
1940	n/a	Py_UCS2 *ucs2 = start;
1941	n/a	assert(index <= PyUnicode_GET_LENGTH(unicode));
1942	n/a
1943	n/a	for (; str < end; ++ucs2, ++str)
1944	n/a	ucs2 = (Py_UCS2)str;
1945	n/a
1946	n/a	assert((ucs2 - start) <= PyUnicode_GET_LENGTH(unicode));
1947	n/a	break;
1948	n/a	}
1949	n/a	default: {
1950	n/a	Py_UCS4 start = (Py_UCS4 )data + index;
1951	n/a	Py_UCS4 *ucs4 = start;
1952	n/a	assert(kind == PyUnicode_4BYTE_KIND);
1953	n/a	assert(index <= PyUnicode_GET_LENGTH(unicode));
1954	n/a
1955	n/a	for (; str < end; ++ucs4, ++str)
1956	n/a	ucs4 = (Py_UCS4)str;
1957	n/a
1958	n/a	assert((ucs4 - start) <= PyUnicode_GET_LENGTH(unicode));
1959	n/a	}
1960	n/a	}
1961	n/a	}
1962	n/a
1963	n/a	static PyObject*
1964	n/a	get_latin1_char(unsigned char ch)
1965	n/a	{
1966	n/a	PyObject *unicode = unicode_latin1[ch];
1967	n/a	if (!unicode) {
1968	n/a	unicode = PyUnicode_New(1, ch);
1969	n/a	if (!unicode)
1970	n/a	return NULL;
1971	n/a	PyUnicode_1BYTE_DATA(unicode)[0] = ch;
1972	n/a	assert(_PyUnicode_CheckConsistency(unicode, 1));
1973	n/a	unicode_latin1[ch] = unicode;
1974	n/a	}
1975	n/a	Py_INCREF(unicode);
1976	n/a	return unicode;
1977	n/a	}
1978	n/a
1979	n/a	static PyObject*
1980	n/a	unicode_char(Py_UCS4 ch)
1981	n/a	{
1982	n/a	PyObject *unicode;
1983	n/a
1984	n/a	assert(ch <= MAX_UNICODE);
1985	n/a
1986	n/a	if (ch < 256)
1987	n/a	return get_latin1_char(ch);
1988	n/a
1989	n/a	unicode = PyUnicode_New(1, ch);
1990	n/a	if (unicode == NULL)
1991	n/a	return NULL;
1992	n/a
1993	n/a	assert(PyUnicode_KIND(unicode) != PyUnicode_1BYTE_KIND);
1994	n/a	if (PyUnicode_KIND(unicode) == PyUnicode_2BYTE_KIND) {
1995	n/a	PyUnicode_2BYTE_DATA(unicode)[0] = (Py_UCS2)ch;
1996	n/a	} else {
1997	n/a	assert(PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND);
1998	n/a	PyUnicode_4BYTE_DATA(unicode)[0] = ch;
1999	n/a	}
2000	n/a	assert(_PyUnicode_CheckConsistency(unicode, 1));
2001	n/a	return unicode;
2002	n/a	}
2003	n/a
2004	n/a	PyObject *
2005	n/a	PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size)
2006	n/a	{
2007	n/a	if (u == NULL)
2008	n/a	return (PyObject*)_PyUnicode_New(size);
2009	n/a
2010	n/a	if (size < 0) {
2011	n/a	PyErr_BadInternalCall();
2012	n/a	return NULL;
2013	n/a	}
2014	n/a
2015	n/a	return PyUnicode_FromWideChar(u, size);
2016	n/a	}
2017	n/a
2018	n/a	PyObject *
2019	n/a	PyUnicode_FromWideChar(const wchar_t *u, Py_ssize_t size)
2020	n/a	{
2021	n/a	PyObject *unicode;
2022	n/a	Py_UCS4 maxchar = 0;
2023	n/a	Py_ssize_t num_surrogates;
2024	n/a
2025	n/a	if (u == NULL && size != 0) {
2026	n/a	PyErr_BadInternalCall();
2027	n/a	return NULL;
2028	n/a	}
2029	n/a
2030	n/a	if (size == -1) {
2031	n/a	size = wcslen(u);
2032	n/a	}
2033	n/a
2034	n/a	/* If the Unicode data is known at construction time, we can apply
2035	n/a	some optimizations which share commonly used objects. */
2036	n/a
2037	n/a	/* Optimization for empty strings */
2038	n/a	if (size == 0)
2039	n/a	_Py_RETURN_UNICODE_EMPTY();
2040	n/a
2041	n/a	/* Single character Unicode objects in the Latin-1 range are
2042	n/a	shared when using this constructor */
2043	n/a	if (size == 1 && (Py_UCS4)*u < 256)
2044	n/a	return get_latin1_char((unsigned char)*u);
2045	n/a
2046	n/a	/* If not empty and not single character, copy the Unicode data
2047	n/a	into the new object */
2048	n/a	if (find_maxchar_surrogates(u, u + size,
2049	n/a	&maxchar, &num_surrogates) == -1)
2050	n/a	return NULL;
2051	n/a
2052	n/a	unicode = PyUnicode_New(size - num_surrogates, maxchar);
2053	n/a	if (!unicode)
2054	n/a	return NULL;
2055	n/a
2056	n/a	switch (PyUnicode_KIND(unicode)) {
2057	n/a	case PyUnicode_1BYTE_KIND:
2058	n/a	_PyUnicode_CONVERT_BYTES(Py_UNICODE, unsigned char,
2059	n/a	u, u + size, PyUnicode_1BYTE_DATA(unicode));
2060	n/a	break;
2061	n/a	case PyUnicode_2BYTE_KIND:
2062	n/a	#if Py_UNICODE_SIZE == 2
2063	n/a	memcpy(PyUnicode_2BYTE_DATA(unicode), u, size * 2);
2064	n/a	#else
2065	n/a	_PyUnicode_CONVERT_BYTES(Py_UNICODE, Py_UCS2,
2066	n/a	u, u + size, PyUnicode_2BYTE_DATA(unicode));
2067	n/a	#endif
2068	n/a	break;
2069	n/a	case PyUnicode_4BYTE_KIND:
2070	n/a	#if SIZEOF_WCHAR_T == 2
2071	n/a	/* This is the only case which has to process surrogates, thus
2072	n/a	a simple copy loop is not enough and we need a function. */
2073	n/a	unicode_convert_wchar_to_ucs4(u, u + size, unicode);
2074	n/a	#else
2075	n/a	assert(num_surrogates == 0);
2076	n/a	memcpy(PyUnicode_4BYTE_DATA(unicode), u, size * 4);
2077	n/a	#endif
2078	n/a	break;
2079	n/a	default:
2080	n/a	assert(0 && "Impossible state");
2081	n/a	}
2082	n/a
2083	n/a	return unicode_result(unicode);
2084	n/a	}
2085	n/a
2086	n/a	PyObject *
2087	n/a	PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size)
2088	n/a	{
2089	n/a	if (size < 0) {
2090	n/a	PyErr_SetString(PyExc_SystemError,
2091	n/a	"Negative size passed to PyUnicode_FromStringAndSize");
2092	n/a	return NULL;
2093	n/a	}
2094	n/a	if (u != NULL)
2095	n/a	return PyUnicode_DecodeUTF8Stateful(u, size, NULL, NULL);
2096	n/a	else
2097	n/a	return (PyObject *)_PyUnicode_New(size);
2098	n/a	}
2099	n/a
2100	n/a	PyObject *
2101	n/a	PyUnicode_FromString(const char *u)
2102	n/a	{
2103	n/a	size_t size = strlen(u);
2104	n/a	if (size > PY_SSIZE_T_MAX) {
2105	n/a	PyErr_SetString(PyExc_OverflowError, "input too long");
2106	n/a	return NULL;
2107	n/a	}
2108	n/a	return PyUnicode_DecodeUTF8Stateful(u, (Py_ssize_t)size, NULL, NULL);
2109	n/a	}
2110	n/a
2111	n/a	PyObject *
2112	n/a	_PyUnicode_FromId(_Py_Identifier *id)
2113	n/a	{
2114	n/a	if (!id->object) {
2115	n/a	id->object = PyUnicode_DecodeUTF8Stateful(id->string,
2116	n/a	strlen(id->string),
2117	n/a	NULL, NULL);
2118	n/a	if (!id->object)
2119	n/a	return NULL;
2120	n/a	PyUnicode_InternInPlace(&id->object);
2121	n/a	assert(!id->next);
2122	n/a	id->next = static_strings;
2123	n/a	static_strings = id;
2124	n/a	}
2125	n/a	return id->object;
2126	n/a	}
2127	n/a
2128	n/a	void
2129	n/a	_PyUnicode_ClearStaticStrings()
2130	n/a	{
2131	n/a	_Py_Identifier tmp, s = static_strings;
2132	n/a	while (s) {
2133	n/a	Py_CLEAR(s->object);
2134	n/a	tmp = s->next;
2135	n/a	s->next = NULL;
2136	n/a	s = tmp;
2137	n/a	}
2138	n/a	static_strings = NULL;
2139	n/a	}
2140	n/a
2141	n/a	/* Internal function, doesn't check maximum character */
2142	n/a
2143	n/a	PyObject*
2144	n/a	_PyUnicode_FromASCII(const char *buffer, Py_ssize_t size)
2145	n/a	{
2146	n/a	const unsigned char s = (const unsigned char )buffer;
2147	n/a	PyObject *unicode;
2148	n/a	if (size == 1) {
2149	n/a	#ifdef Py_DEBUG
2150	n/a	assert((unsigned char)s[0] < 128);
2151	n/a	#endif
2152	n/a	return get_latin1_char(s[0]);
2153	n/a	}
2154	n/a	unicode = PyUnicode_New(size, 127);
2155	n/a	if (!unicode)
2156	n/a	return NULL;
2157	n/a	memcpy(PyUnicode_1BYTE_DATA(unicode), s, size);
2158	n/a	assert(_PyUnicode_CheckConsistency(unicode, 1));
2159	n/a	return unicode;
2160	n/a	}
2161	n/a
2162	n/a	static Py_UCS4
2163	n/a	kind_maxchar_limit(unsigned int kind)
2164	n/a	{
2165	n/a	switch (kind) {
2166	n/a	case PyUnicode_1BYTE_KIND:
2167	n/a	return 0x80;
2168	n/a	case PyUnicode_2BYTE_KIND:
2169	n/a	return 0x100;
2170	n/a	case PyUnicode_4BYTE_KIND:
2171	n/a	return 0x10000;
2172	n/a	default:
2173	n/a	assert(0 && "invalid kind");
2174	n/a	return MAX_UNICODE;
2175	n/a	}
2176	n/a	}
2177	n/a
2178	n/a	static inline Py_UCS4
2179	n/a	align_maxchar(Py_UCS4 maxchar)
2180	n/a	{
2181	n/a	if (maxchar <= 127)
2182	n/a	return 127;
2183	n/a	else if (maxchar <= 255)
2184	n/a	return 255;
2185	n/a	else if (maxchar <= 65535)
2186	n/a	return 65535;
2187	n/a	else
2188	n/a	return MAX_UNICODE;
2189	n/a	}
2190	n/a
2191	n/a	static PyObject*
2192	n/a	_PyUnicode_FromUCS1(const Py_UCS1* u, Py_ssize_t size)
2193	n/a	{
2194	n/a	PyObject *res;
2195	n/a	unsigned char max_char;
2196	n/a
2197	n/a	if (size == 0)
2198	n/a	_Py_RETURN_UNICODE_EMPTY();
2199	n/a	assert(size > 0);
2200	n/a	if (size == 1)
2201	n/a	return get_latin1_char(u[0]);
2202	n/a
2203	n/a	max_char = ucs1lib_find_max_char(u, u + size);
2204	n/a	res = PyUnicode_New(size, max_char);
2205	n/a	if (!res)
2206	n/a	return NULL;
2207	n/a	memcpy(PyUnicode_1BYTE_DATA(res), u, size);
2208	n/a	assert(_PyUnicode_CheckConsistency(res, 1));
2209	n/a	return res;
2210	n/a	}
2211	n/a
2212	n/a	static PyObject*
2213	n/a	_PyUnicode_FromUCS2(const Py_UCS2 *u, Py_ssize_t size)
2214	n/a	{
2215	n/a	PyObject *res;
2216	n/a	Py_UCS2 max_char;
2217	n/a
2218	n/a	if (size == 0)
2219	n/a	_Py_RETURN_UNICODE_EMPTY();
2220	n/a	assert(size > 0);
2221	n/a	if (size == 1)
2222	n/a	return unicode_char(u[0]);
2223	n/a
2224	n/a	max_char = ucs2lib_find_max_char(u, u + size);
2225	n/a	res = PyUnicode_New(size, max_char);
2226	n/a	if (!res)
2227	n/a	return NULL;
2228	n/a	if (max_char >= 256)
2229	n/a	memcpy(PyUnicode_2BYTE_DATA(res), u, sizeof(Py_UCS2)*size);
2230	n/a	else {
2231	n/a	_PyUnicode_CONVERT_BYTES(
2232	n/a	Py_UCS2, Py_UCS1, u, u + size, PyUnicode_1BYTE_DATA(res));
2233	n/a	}
2234	n/a	assert(_PyUnicode_CheckConsistency(res, 1));
2235	n/a	return res;
2236	n/a	}
2237	n/a
2238	n/a	static PyObject*
2239	n/a	_PyUnicode_FromUCS4(const Py_UCS4 *u, Py_ssize_t size)
2240	n/a	{
2241	n/a	PyObject *res;
2242	n/a	Py_UCS4 max_char;
2243	n/a
2244	n/a	if (size == 0)
2245	n/a	_Py_RETURN_UNICODE_EMPTY();
2246	n/a	assert(size > 0);
2247	n/a	if (size == 1)
2248	n/a	return unicode_char(u[0]);
2249	n/a
2250	n/a	max_char = ucs4lib_find_max_char(u, u + size);
2251	n/a	res = PyUnicode_New(size, max_char);
2252	n/a	if (!res)
2253	n/a	return NULL;
2254	n/a	if (max_char < 256)
2255	n/a	_PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS1, u, u + size,
2256	n/a	PyUnicode_1BYTE_DATA(res));
2257	n/a	else if (max_char < 0x10000)
2258	n/a	_PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS2, u, u + size,
2259	n/a	PyUnicode_2BYTE_DATA(res));
2260	n/a	else
2261	n/a	memcpy(PyUnicode_4BYTE_DATA(res), u, sizeof(Py_UCS4)*size);
2262	n/a	assert(_PyUnicode_CheckConsistency(res, 1));
2263	n/a	return res;
2264	n/a	}
2265	n/a
2266	n/a	PyObject*
2267	n/a	PyUnicode_FromKindAndData(int kind, const void *buffer, Py_ssize_t size)
2268	n/a	{
2269	n/a	if (size < 0) {
2270	n/a	PyErr_SetString(PyExc_ValueError, "size must be positive");
2271	n/a	return NULL;
2272	n/a	}
2273	n/a	switch (kind) {
2274	n/a	case PyUnicode_1BYTE_KIND:
2275	n/a	return _PyUnicode_FromUCS1(buffer, size);
2276	n/a	case PyUnicode_2BYTE_KIND:
2277	n/a	return _PyUnicode_FromUCS2(buffer, size);
2278	n/a	case PyUnicode_4BYTE_KIND:
2279	n/a	return _PyUnicode_FromUCS4(buffer, size);
2280	n/a	default:
2281	n/a	PyErr_SetString(PyExc_SystemError, "invalid kind");
2282	n/a	return NULL;
2283	n/a	}
2284	n/a	}
2285	n/a
2286	n/a	Py_UCS4
2287	n/a	_PyUnicode_FindMaxChar(PyObject *unicode, Py_ssize_t start, Py_ssize_t end)
2288	n/a	{
2289	n/a	enum PyUnicode_Kind kind;
2290	n/a	void startptr, endptr;
2291	n/a
2292	n/a	assert(PyUnicode_IS_READY(unicode));
2293	n/a	assert(0 <= start);
2294	n/a	assert(end <= PyUnicode_GET_LENGTH(unicode));
2295	n/a	assert(start <= end);
2296	n/a
2297	n/a	if (start == 0 && end == PyUnicode_GET_LENGTH(unicode))
2298	n/a	return PyUnicode_MAX_CHAR_VALUE(unicode);
2299	n/a
2300	n/a	if (start == end)
2301	n/a	return 127;
2302	n/a
2303	n/a	if (PyUnicode_IS_ASCII(unicode))
2304	n/a	return 127;
2305	n/a
2306	n/a	kind = PyUnicode_KIND(unicode);
2307	n/a	startptr = PyUnicode_DATA(unicode);
2308	n/a	endptr = (char )startptr + end kind;
2309	n/a	startptr = (char )startptr + start kind;
2310	n/a	switch(kind) {
2311	n/a	case PyUnicode_1BYTE_KIND:
2312	n/a	return ucs1lib_find_max_char(startptr, endptr);
2313	n/a	case PyUnicode_2BYTE_KIND:
2314	n/a	return ucs2lib_find_max_char(startptr, endptr);
2315	n/a	case PyUnicode_4BYTE_KIND:
2316	n/a	return ucs4lib_find_max_char(startptr, endptr);
2317	n/a	default:
2318	n/a	assert(0);
2319	n/a	return 0;
2320	n/a	}
2321	n/a	}
2322	n/a
2323	n/a	/* Ensure that a string uses the most efficient storage, if it is not the
2324	n/a	case: create a new string with of the right kind. Write NULL into *p_unicode
2325	n/a	on error. */
2326	n/a	static void
2327	n/a	unicode_adjust_maxchar(PyObject **p_unicode)
2328	n/a	{
2329	n/a	PyObject unicode, copy;
2330	n/a	Py_UCS4 max_char;
2331	n/a	Py_ssize_t len;
2332	n/a	unsigned int kind;
2333	n/a
2334	n/a	assert(p_unicode != NULL);
2335	n/a	unicode = *p_unicode;
2336	n/a	assert(PyUnicode_IS_READY(unicode));
2337	n/a	if (PyUnicode_IS_ASCII(unicode))
2338	n/a	return;
2339	n/a
2340	n/a	len = PyUnicode_GET_LENGTH(unicode);
2341	n/a	kind = PyUnicode_KIND(unicode);
2342	n/a	if (kind == PyUnicode_1BYTE_KIND) {
2343	n/a	const Py_UCS1 *u = PyUnicode_1BYTE_DATA(unicode);
2344	n/a	max_char = ucs1lib_find_max_char(u, u + len);
2345	n/a	if (max_char >= 128)
2346	n/a	return;
2347	n/a	}
2348	n/a	else if (kind == PyUnicode_2BYTE_KIND) {
2349	n/a	const Py_UCS2 *u = PyUnicode_2BYTE_DATA(unicode);
2350	n/a	max_char = ucs2lib_find_max_char(u, u + len);
2351	n/a	if (max_char >= 256)
2352	n/a	return;
2353	n/a	}
2354	n/a	else {
2355	n/a	const Py_UCS4 *u = PyUnicode_4BYTE_DATA(unicode);
2356	n/a	assert(kind == PyUnicode_4BYTE_KIND);
2357	n/a	max_char = ucs4lib_find_max_char(u, u + len);
2358	n/a	if (max_char >= 0x10000)
2359	n/a	return;
2360	n/a	}
2361	n/a	copy = PyUnicode_New(len, max_char);
2362	n/a	if (copy != NULL)
2363	n/a	_PyUnicode_FastCopyCharacters(copy, 0, unicode, 0, len);
2364	n/a	Py_DECREF(unicode);
2365	n/a	*p_unicode = copy;
2366	n/a	}
2367	n/a
2368	n/a	PyObject*
2369	n/a	_PyUnicode_Copy(PyObject *unicode)
2370	n/a	{
2371	n/a	Py_ssize_t length;
2372	n/a	PyObject *copy;
2373	n/a
2374	n/a	if (!PyUnicode_Check(unicode)) {
2375	n/a	PyErr_BadInternalCall();
2376	n/a	return NULL;
2377	n/a	}
2378	n/a	if (PyUnicode_READY(unicode) == -1)
2379	n/a	return NULL;
2380	n/a
2381	n/a	length = PyUnicode_GET_LENGTH(unicode);
2382	n/a	copy = PyUnicode_New(length, PyUnicode_MAX_CHAR_VALUE(unicode));
2383	n/a	if (!copy)
2384	n/a	return NULL;
2385	n/a	assert(PyUnicode_KIND(copy) == PyUnicode_KIND(unicode));
2386	n/a
2387	n/a	memcpy(PyUnicode_DATA(copy), PyUnicode_DATA(unicode),
2388	n/a	length * PyUnicode_KIND(unicode));
2389	n/a	assert(_PyUnicode_CheckConsistency(copy, 1));
2390	n/a	return copy;
2391	n/a	}
2392	n/a
2393	n/a
2394	n/a	/* Widen Unicode objects to larger buffers. Don't write terminating null
2395	n/a	character. Return NULL on error. */
2396	n/a
2397	n/a	void*
2398	n/a	_PyUnicode_AsKind(PyObject *s, unsigned int kind)
2399	n/a	{
2400	n/a	Py_ssize_t len;
2401	n/a	void *result;
2402	n/a	unsigned int skind;
2403	n/a
2404	n/a	if (PyUnicode_READY(s) == -1)
2405	n/a	return NULL;
2406	n/a
2407	n/a	len = PyUnicode_GET_LENGTH(s);
2408	n/a	skind = PyUnicode_KIND(s);
2409	n/a	if (skind >= kind) {
2410	n/a	PyErr_SetString(PyExc_SystemError, "invalid widening attempt");
2411	n/a	return NULL;
2412	n/a	}
2413	n/a	switch (kind) {
2414	n/a	case PyUnicode_2BYTE_KIND:
2415	n/a	result = PyMem_New(Py_UCS2, len);
2416	n/a	if (!result)
2417	n/a	return PyErr_NoMemory();
2418	n/a	assert(skind == PyUnicode_1BYTE_KIND);
2419	n/a	_PyUnicode_CONVERT_BYTES(
2420	n/a	Py_UCS1, Py_UCS2,
2421	n/a	PyUnicode_1BYTE_DATA(s),
2422	n/a	PyUnicode_1BYTE_DATA(s) + len,
2423	n/a	result);
2424	n/a	return result;
2425	n/a	case PyUnicode_4BYTE_KIND:
2426	n/a	result = PyMem_New(Py_UCS4, len);
2427	n/a	if (!result)
2428	n/a	return PyErr_NoMemory();
2429	n/a	if (skind == PyUnicode_2BYTE_KIND) {
2430	n/a	_PyUnicode_CONVERT_BYTES(
2431	n/a	Py_UCS2, Py_UCS4,
2432	n/a	PyUnicode_2BYTE_DATA(s),
2433	n/a	PyUnicode_2BYTE_DATA(s) + len,
2434	n/a	result);
2435	n/a	}
2436	n/a	else {
2437	n/a	assert(skind == PyUnicode_1BYTE_KIND);
2438	n/a	_PyUnicode_CONVERT_BYTES(
2439	n/a	Py_UCS1, Py_UCS4,
2440	n/a	PyUnicode_1BYTE_DATA(s),
2441	n/a	PyUnicode_1BYTE_DATA(s) + len,
2442	n/a	result);
2443	n/a	}
2444	n/a	return result;
2445	n/a	default:
2446	n/a	break;
2447	n/a	}
2448	n/a	PyErr_SetString(PyExc_SystemError, "invalid kind");
2449	n/a	return NULL;
2450	n/a	}
2451	n/a
2452	n/a	static Py_UCS4*
2453	n/a	as_ucs4(PyObject string, Py_UCS4 target, Py_ssize_t targetsize,
2454	n/a	int copy_null)
2455	n/a	{
2456	n/a	int kind;
2457	n/a	void *data;
2458	n/a	Py_ssize_t len, targetlen;
2459	n/a	if (PyUnicode_READY(string) == -1)
2460	n/a	return NULL;
2461	n/a	kind = PyUnicode_KIND(string);
2462	n/a	data = PyUnicode_DATA(string);
2463	n/a	len = PyUnicode_GET_LENGTH(string);
2464	n/a	targetlen = len;
2465	n/a	if (copy_null)
2466	n/a	targetlen++;
2467	n/a	if (!target) {
2468	n/a	target = PyMem_New(Py_UCS4, targetlen);
2469	n/a	if (!target) {
2470	n/a	PyErr_NoMemory();
2471	n/a	return NULL;
2472	n/a	}
2473	n/a	}
2474	n/a	else {
2475	n/a	if (targetsize < targetlen) {
2476	n/a	PyErr_Format(PyExc_SystemError,
2477	n/a	"string is longer than the buffer");
2478	n/a	if (copy_null && 0 < targetsize)
2479	n/a	target[0] = 0;
2480	n/a	return NULL;
2481	n/a	}
2482	n/a	}
2483	n/a	if (kind == PyUnicode_1BYTE_KIND) {
2484	n/a	Py_UCS1 start = (Py_UCS1 ) data;
2485	n/a	_PyUnicode_CONVERT_BYTES(Py_UCS1, Py_UCS4, start, start + len, target);
2486	n/a	}
2487	n/a	else if (kind == PyUnicode_2BYTE_KIND) {
2488	n/a	Py_UCS2 start = (Py_UCS2 ) data;
2489	n/a	_PyUnicode_CONVERT_BYTES(Py_UCS2, Py_UCS4, start, start + len, target);
2490	n/a	}
2491	n/a	else {
2492	n/a	assert(kind == PyUnicode_4BYTE_KIND);
2493	n/a	memcpy(target, data, len * sizeof(Py_UCS4));
2494	n/a	}
2495	n/a	if (copy_null)
2496	n/a	target[len] = 0;
2497	n/a	return target;
2498	n/a	}
2499	n/a
2500	n/a	Py_UCS4*
2501	n/a	PyUnicode_AsUCS4(PyObject string, Py_UCS4 target, Py_ssize_t targetsize,
2502	n/a	int copy_null)
2503	n/a	{
2504	n/a	if (target == NULL \|\| targetsize < 0) {
2505	n/a	PyErr_BadInternalCall();
2506	n/a	return NULL;
2507	n/a	}
2508	n/a	return as_ucs4(string, target, targetsize, copy_null);
2509	n/a	}
2510	n/a
2511	n/a	Py_UCS4*
2512	n/a	PyUnicode_AsUCS4Copy(PyObject *string)
2513	n/a	{
2514	n/a	return as_ucs4(string, NULL, 0, 1);
2515	n/a	}
2516	n/a
2517	n/a	/* maximum number of characters required for output of %lld or %p.
2518	n/a	We need at most ceil(log10(256)*SIZEOF_LONG_LONG) digits,
2519	n/a	plus 1 for the sign. 53/22 is an upper bound for log10(256). */
2520	n/a	#define MAX_LONG_LONG_CHARS (2 + (SIZEOF_LONG_LONG*53-1) / 22)
2521	n/a
2522	n/a	static int
2523	n/a	unicode_fromformat_write_str(_PyUnicodeWriter writer, PyObject str,
2524	n/a	Py_ssize_t width, Py_ssize_t precision)
2525	n/a	{
2526	n/a	Py_ssize_t length, fill, arglen;
2527	n/a	Py_UCS4 maxchar;
2528	n/a
2529	n/a	if (PyUnicode_READY(str) == -1)
2530	n/a	return -1;
2531	n/a
2532	n/a	length = PyUnicode_GET_LENGTH(str);
2533	n/a	if ((precision == -1 \|\| precision >= length)
2534	n/a	&& width <= length)
2535	n/a	return _PyUnicodeWriter_WriteStr(writer, str);
2536	n/a
2537	n/a	if (precision != -1)
2538	n/a	length = Py_MIN(precision, length);
2539	n/a
2540	n/a	arglen = Py_MAX(length, width);
2541	n/a	if (PyUnicode_MAX_CHAR_VALUE(str) > writer->maxchar)
2542	n/a	maxchar = _PyUnicode_FindMaxChar(str, 0, length);
2543	n/a	else
2544	n/a	maxchar = writer->maxchar;
2545	n/a
2546	n/a	if (_PyUnicodeWriter_Prepare(writer, arglen, maxchar) == -1)
2547	n/a	return -1;
2548	n/a
2549	n/a	if (width > length) {
2550	n/a	fill = width - length;
2551	n/a	if (PyUnicode_Fill(writer->buffer, writer->pos, fill, ' ') == -1)
2552	n/a	return -1;
2553	n/a	writer->pos += fill;
2554	n/a	}
2555	n/a
2556	n/a	_PyUnicode_FastCopyCharacters(writer->buffer, writer->pos,
2557	n/a	str, 0, length);
2558	n/a	writer->pos += length;
2559	n/a	return 0;
2560	n/a	}
2561	n/a
2562	n/a	static int
2563	n/a	unicode_fromformat_write_cstr(_PyUnicodeWriter writer, const char str,
2564	n/a	Py_ssize_t width, Py_ssize_t precision)
2565	n/a	{
2566	n/a	/* UTF-8 */
2567	n/a	Py_ssize_t length;
2568	n/a	PyObject *unicode;
2569	n/a	int res;
2570	n/a
2571	n/a	length = strlen(str);
2572	n/a	if (precision != -1)
2573	n/a	length = Py_MIN(length, precision);
2574	n/a	unicode = PyUnicode_DecodeUTF8Stateful(str, length, "replace", NULL);
2575	n/a	if (unicode == NULL)
2576	n/a	return -1;
2577	n/a
2578	n/a	res = unicode_fromformat_write_str(writer, unicode, width, -1);
2579	n/a	Py_DECREF(unicode);
2580	n/a	return res;
2581	n/a	}
2582	n/a
2583	n/a	static const char*
2584	n/a	unicode_fromformat_arg(_PyUnicodeWriter *writer,
2585	n/a	const char f, va_list vargs)
2586	n/a	{
2587	n/a	const char *p;
2588	n/a	Py_ssize_t len;
2589	n/a	int zeropad;
2590	n/a	Py_ssize_t width;
2591	n/a	Py_ssize_t precision;
2592	n/a	int longflag;
2593	n/a	int longlongflag;
2594	n/a	int size_tflag;
2595	n/a	Py_ssize_t fill;
2596	n/a
2597	n/a	p = f;
2598	n/a	f++;
2599	n/a	zeropad = 0;
2600	n/a	if (*f == '0') {
2601	n/a	zeropad = 1;
2602	n/a	f++;
2603	n/a	}
2604	n/a
2605	n/a	/* parse the width.precision part, e.g. "%2.5s" => width=2, precision=5 */
2606	n/a	width = -1;
2607	n/a	if (Py_ISDIGIT((unsigned)*f)) {
2608	n/a	width = *f - '0';
2609	n/a	f++;
2610	n/a	while (Py_ISDIGIT((unsigned)*f)) {
2611	n/a	if (width > (PY_SSIZE_T_MAX - ((int)*f - '0')) / 10) {
2612	n/a	PyErr_SetString(PyExc_ValueError,
2613	n/a	"width too big");
2614	n/a	return NULL;
2615	n/a	}
2616	n/a	width = (width * 10) + (*f - '0');
2617	n/a	f++;
2618	n/a	}
2619	n/a	}
2620	n/a	precision = -1;
2621	n/a	if (*f == '.') {
2622	n/a	f++;
2623	n/a	if (Py_ISDIGIT((unsigned)*f)) {
2624	n/a	precision = (*f - '0');
2625	n/a	f++;
2626	n/a	while (Py_ISDIGIT((unsigned)*f)) {
2627	n/a	if (precision > (PY_SSIZE_T_MAX - ((int)*f - '0')) / 10) {
2628	n/a	PyErr_SetString(PyExc_ValueError,
2629	n/a	"precision too big");
2630	n/a	return NULL;
2631	n/a	}
2632	n/a	precision = (precision * 10) + (*f - '0');
2633	n/a	f++;
2634	n/a	}
2635	n/a	}
2636	n/a	if (*f == '%') {
2637	n/a	/* "%.3%s" => f points to "3" */
2638	n/a	f--;
2639	n/a	}
2640	n/a	}
2641	n/a	if (*f == '\0') {
2642	n/a	/* bogus format "%.123" => go backward, f points to "3" */
2643	n/a	f--;
2644	n/a	}
2645	n/a
2646	n/a	/* Handle %ld, %lu, %lld and %llu. */
2647	n/a	longflag = 0;
2648	n/a	longlongflag = 0;
2649	n/a	size_tflag = 0;
2650	n/a	if (*f == 'l') {
2651	n/a	if (f[1] == 'd' \|\| f[1] == 'u' \|\| f[1] == 'i') {
2652	n/a	longflag = 1;
2653	n/a	++f;
2654	n/a	}
2655	n/a	else if (f[1] == 'l' &&
2656	n/a	(f[2] == 'd' \|\| f[2] == 'u' \|\| f[2] == 'i')) {
2657	n/a	longlongflag = 1;
2658	n/a	f += 2;
2659	n/a	}
2660	n/a	}
2661	n/a	/* handle the size_t flag. */
2662	n/a	else if (*f == 'z' && (f[1] == 'd' \|\| f[1] == 'u' \|\| f[1] == 'i')) {
2663	n/a	size_tflag = 1;
2664	n/a	++f;
2665	n/a	}
2666	n/a
2667	n/a	if (f[1] == '\0')
2668	n/a	writer->overallocate = 0;
2669	n/a
2670	n/a	switch (*f) {
2671	n/a	case 'c':
2672	n/a	{
2673	n/a	int ordinal = va_arg(*vargs, int);
2674	n/a	if (ordinal < 0 \|\| ordinal > MAX_UNICODE) {
2675	n/a	PyErr_SetString(PyExc_OverflowError,
2676	n/a	"character argument not in range(0x110000)");
2677	n/a	return NULL;
2678	n/a	}
2679	n/a	if (_PyUnicodeWriter_WriteCharInline(writer, ordinal) < 0)
2680	n/a	return NULL;
2681	n/a	break;
2682	n/a	}
2683	n/a
2684	n/a	case 'i':
2685	n/a	case 'd':
2686	n/a	case 'u':
2687	n/a	case 'x':
2688	n/a	{
2689	n/a	/* used by sprintf */
2690	n/a	char buffer[MAX_LONG_LONG_CHARS];
2691	n/a	Py_ssize_t arglen;
2692	n/a
2693	n/a	if (*f == 'u') {
2694	n/a	if (longflag)
2695	n/a	len = sprintf(buffer, "%lu",
2696	n/a	va_arg(*vargs, unsigned long));
2697	n/a	else if (longlongflag)
2698	n/a	len = sprintf(buffer, "%llu",
2699	n/a	va_arg(*vargs, unsigned long long));
2700	n/a	else if (size_tflag)
2701	n/a	len = sprintf(buffer, "%" PY_FORMAT_SIZE_T "u",
2702	n/a	va_arg(*vargs, size_t));
2703	n/a	else
2704	n/a	len = sprintf(buffer, "%u",
2705	n/a	va_arg(*vargs, unsigned int));
2706	n/a	}
2707	n/a	else if (*f == 'x') {
2708	n/a	len = sprintf(buffer, "%x", va_arg(*vargs, int));
2709	n/a	}
2710	n/a	else {
2711	n/a	if (longflag)
2712	n/a	len = sprintf(buffer, "%li",
2713	n/a	va_arg(*vargs, long));
2714	n/a	else if (longlongflag)
2715	n/a	len = sprintf(buffer, "%lli",
2716	n/a	va_arg(*vargs, long long));
2717	n/a	else if (size_tflag)
2718	n/a	len = sprintf(buffer, "%" PY_FORMAT_SIZE_T "i",
2719	n/a	va_arg(*vargs, Py_ssize_t));
2720	n/a	else
2721	n/a	len = sprintf(buffer, "%i",
2722	n/a	va_arg(*vargs, int));
2723	n/a	}
2724	n/a	assert(len >= 0);
2725	n/a
2726	n/a	if (precision < len)
2727	n/a	precision = len;
2728	n/a
2729	n/a	arglen = Py_MAX(precision, width);
2730	n/a	if (_PyUnicodeWriter_Prepare(writer, arglen, 127) == -1)
2731	n/a	return NULL;
2732	n/a
2733	n/a	if (width > precision) {
2734	n/a	Py_UCS4 fillchar;
2735	n/a	fill = width - precision;
2736	n/a	fillchar = zeropad?'0':' ';
2737	n/a	if (PyUnicode_Fill(writer->buffer, writer->pos, fill, fillchar) == -1)
2738	n/a	return NULL;
2739	n/a	writer->pos += fill;
2740	n/a	}
2741	n/a	if (precision > len) {
2742	n/a	fill = precision - len;
2743	n/a	if (PyUnicode_Fill(writer->buffer, writer->pos, fill, '0') == -1)
2744	n/a	return NULL;
2745	n/a	writer->pos += fill;
2746	n/a	}
2747	n/a
2748	n/a	if (_PyUnicodeWriter_WriteASCIIString(writer, buffer, len) < 0)
2749	n/a	return NULL;
2750	n/a	break;
2751	n/a	}
2752	n/a
2753	n/a	case 'p':
2754	n/a	{
2755	n/a	char number[MAX_LONG_LONG_CHARS];
2756	n/a
2757	n/a	len = sprintf(number, "%p", va_arg(vargs, void));
2758	n/a	assert(len >= 0);
2759	n/a
2760	n/a	/* %p is ill-defined: ensure leading 0x. */
2761	n/a	if (number[1] == 'X')
2762	n/a	number[1] = 'x';
2763	n/a	else if (number[1] != 'x') {
2764	n/a	memmove(number + 2, number,
2765	n/a	strlen(number) + 1);
2766	n/a	number[0] = '0';
2767	n/a	number[1] = 'x';
2768	n/a	len += 2;
2769	n/a	}
2770	n/a
2771	n/a	if (_PyUnicodeWriter_WriteASCIIString(writer, number, len) < 0)
2772	n/a	return NULL;
2773	n/a	break;
2774	n/a	}
2775	n/a
2776	n/a	case 's':
2777	n/a	{
2778	n/a	/* UTF-8 */
2779	n/a	const char s = va_arg(vargs, const char*);
2780	n/a	if (unicode_fromformat_write_cstr(writer, s, width, precision) < 0)
2781	n/a	return NULL;
2782	n/a	break;
2783	n/a	}
2784	n/a
2785	n/a	case 'U':
2786	n/a	{
2787	n/a	PyObject obj = va_arg(vargs, PyObject *);
2788	n/a	assert(obj && _PyUnicode_CHECK(obj));
2789	n/a
2790	n/a	if (unicode_fromformat_write_str(writer, obj, width, precision) == -1)
2791	n/a	return NULL;
2792	n/a	break;
2793	n/a	}
2794	n/a
2795	n/a	case 'V':
2796	n/a	{
2797	n/a	PyObject obj = va_arg(vargs, PyObject *);
2798	n/a	const char str = va_arg(vargs, const char *);
2799	n/a	if (obj) {
2800	n/a	assert(_PyUnicode_CHECK(obj));
2801	n/a	if (unicode_fromformat_write_str(writer, obj, width, precision) == -1)
2802	n/a	return NULL;
2803	n/a	}
2804	n/a	else {
2805	n/a	assert(str != NULL);
2806	n/a	if (unicode_fromformat_write_cstr(writer, str, width, precision) < 0)
2807	n/a	return NULL;
2808	n/a	}
2809	n/a	break;
2810	n/a	}
2811	n/a
2812	n/a	case 'S':
2813	n/a	{
2814	n/a	PyObject obj = va_arg(vargs, PyObject *);
2815	n/a	PyObject *str;
2816	n/a	assert(obj);
2817	n/a	str = PyObject_Str(obj);
2818	n/a	if (!str)
2819	n/a	return NULL;
2820	n/a	if (unicode_fromformat_write_str(writer, str, width, precision) == -1) {
2821	n/a	Py_DECREF(str);
2822	n/a	return NULL;
2823	n/a	}
2824	n/a	Py_DECREF(str);
2825	n/a	break;
2826	n/a	}
2827	n/a
2828	n/a	case 'R':
2829	n/a	{
2830	n/a	PyObject obj = va_arg(vargs, PyObject *);
2831	n/a	PyObject *repr;
2832	n/a	assert(obj);
2833	n/a	repr = PyObject_Repr(obj);
2834	n/a	if (!repr)
2835	n/a	return NULL;
2836	n/a	if (unicode_fromformat_write_str(writer, repr, width, precision) == -1) {
2837	n/a	Py_DECREF(repr);
2838	n/a	return NULL;
2839	n/a	}
2840	n/a	Py_DECREF(repr);
2841	n/a	break;
2842	n/a	}
2843	n/a
2844	n/a	case 'A':
2845	n/a	{
2846	n/a	PyObject obj = va_arg(vargs, PyObject *);
2847	n/a	PyObject *ascii;
2848	n/a	assert(obj);
2849	n/a	ascii = PyObject_ASCII(obj);
2850	n/a	if (!ascii)
2851	n/a	return NULL;
2852	n/a	if (unicode_fromformat_write_str(writer, ascii, width, precision) == -1) {
2853	n/a	Py_DECREF(ascii);
2854	n/a	return NULL;
2855	n/a	}
2856	n/a	Py_DECREF(ascii);
2857	n/a	break;
2858	n/a	}
2859	n/a
2860	n/a	case '%':
2861	n/a	if (_PyUnicodeWriter_WriteCharInline(writer, '%') < 0)
2862	n/a	return NULL;
2863	n/a	break;
2864	n/a
2865	n/a	default:
2866	n/a	/* if we stumble upon an unknown formatting code, copy the rest
2867	n/a	of the format string to the output string. (we cannot just
2868	n/a	skip the code, since there's no way to know what's in the
2869	n/a	argument list) */
2870	n/a	len = strlen(p);
2871	n/a	if (_PyUnicodeWriter_WriteLatin1String(writer, p, len) == -1)
2872	n/a	return NULL;
2873	n/a	f = p+len;
2874	n/a	return f;
2875	n/a	}
2876	n/a
2877	n/a	f++;
2878	n/a	return f;
2879	n/a	}
2880	n/a
2881	n/a	PyObject *
2882	n/a	PyUnicode_FromFormatV(const char *format, va_list vargs)
2883	n/a	{
2884	n/a	va_list vargs2;
2885	n/a	const char *f;
2886	n/a	_PyUnicodeWriter writer;
2887	n/a
2888	n/a	_PyUnicodeWriter_Init(&writer);
2889	n/a	writer.min_length = strlen(format) + 100;
2890	n/a	writer.overallocate = 1;
2891	n/a
2892	n/a	// Copy varags to be able to pass a reference to a subfunction.
2893	n/a	va_copy(vargs2, vargs);
2894	n/a
2895	n/a	for (f = format; *f; ) {
2896	n/a	if (*f == '%') {
2897	n/a	f = unicode_fromformat_arg(&writer, f, &vargs2);
2898	n/a	if (f == NULL)
2899	n/a	goto fail;
2900	n/a	}
2901	n/a	else {
2902	n/a	const char *p;
2903	n/a	Py_ssize_t len;
2904	n/a
2905	n/a	p = f;
2906	n/a	do
2907	n/a	{
2908	n/a	if ((unsigned char)*p > 127) {
2909	n/a	PyErr_Format(PyExc_ValueError,
2910	n/a	"PyUnicode_FromFormatV() expects an ASCII-encoded format "
2911	n/a	"string, got a non-ASCII byte: 0x%02x",
2912	n/a	(unsigned char)*p);
2913	n/a	goto fail;
2914	n/a	}
2915	n/a	p++;
2916	n/a	}
2917	n/a	while (p != '\0' && p != '%');
2918	n/a	len = p - f;
2919	n/a
2920	n/a	if (*p == '\0')
2921	n/a	writer.overallocate = 0;
2922	n/a
2923	n/a	if (_PyUnicodeWriter_WriteASCIIString(&writer, f, len) < 0)
2924	n/a	goto fail;
2925	n/a
2926	n/a	f = p;
2927	n/a	}
2928	n/a	}
2929	n/a	va_end(vargs2);
2930	n/a	return _PyUnicodeWriter_Finish(&writer);
2931	n/a
2932	n/a	fail:
2933	n/a	va_end(vargs2);
2934	n/a	_PyUnicodeWriter_Dealloc(&writer);
2935	n/a	return NULL;
2936	n/a	}
2937	n/a
2938	n/a	PyObject *
2939	n/a	PyUnicode_FromFormat(const char *format, ...)
2940	n/a	{
2941	n/a	PyObject* ret;
2942	n/a	va_list vargs;
2943	n/a
2944	n/a	#ifdef HAVE_STDARG_PROTOTYPES
2945	n/a	va_start(vargs, format);
2946	n/a	#else
2947	n/a	va_start(vargs);
2948	n/a	#endif
2949	n/a	ret = PyUnicode_FromFormatV(format, vargs);
2950	n/a	va_end(vargs);
2951	n/a	return ret;
2952	n/a	}
2953	n/a
2954	n/a	#ifdef HAVE_WCHAR_H
2955	n/a
2956	n/a	/* Helper function for PyUnicode_AsWideChar() and PyUnicode_AsWideCharString():
2957	n/a	convert a Unicode object to a wide character string.
2958	n/a
2959	n/a	- If w is NULL: return the number of wide characters (including the null
2960	n/a	character) required to convert the unicode object. Ignore size argument.
2961	n/a
2962	n/a	- Otherwise: return the number of wide characters (excluding the null
2963	n/a	character) written into w. Write at most size wide characters (including
2964	n/a	the null character). */
2965	n/a	static Py_ssize_t
2966	n/a	unicode_aswidechar(PyObject *unicode,
2967	n/a	wchar_t *w,
2968	n/a	Py_ssize_t size)
2969	n/a	{
2970	n/a	Py_ssize_t res;
2971	n/a	const wchar_t *wstr;
2972	n/a
2973	n/a	wstr = PyUnicode_AsUnicodeAndSize(unicode, &res);
2974	n/a	if (wstr == NULL)
2975	n/a	return -1;
2976	n/a
2977	n/a	if (w != NULL) {
2978	n/a	if (size > res)
2979	n/a	size = res + 1;
2980	n/a	else
2981	n/a	res = size;
2982	n/a	memcpy(w, wstr, size * sizeof(wchar_t));
2983	n/a	return res;
2984	n/a	}
2985	n/a	else
2986	n/a	return res + 1;
2987	n/a	}
2988	n/a
2989	n/a	Py_ssize_t
2990	n/a	PyUnicode_AsWideChar(PyObject *unicode,
2991	n/a	wchar_t *w,
2992	n/a	Py_ssize_t size)
2993	n/a	{
2994	n/a	if (unicode == NULL) {
2995	n/a	PyErr_BadInternalCall();
2996	n/a	return -1;
2997	n/a	}
2998	n/a	return unicode_aswidechar(unicode, w, size);
2999	n/a	}
3000	n/a
3001	n/a	wchar_t*
3002	n/a	PyUnicode_AsWideCharString(PyObject *unicode,
3003	n/a	Py_ssize_t *size)
3004	n/a	{
3005	n/a	wchar_t* buffer;
3006	n/a	Py_ssize_t buflen;
3007	n/a
3008	n/a	if (unicode == NULL) {
3009	n/a	PyErr_BadInternalCall();
3010	n/a	return NULL;
3011	n/a	}
3012	n/a
3013	n/a	buflen = unicode_aswidechar(unicode, NULL, 0);
3014	n/a	if (buflen == -1)
3015	n/a	return NULL;
3016	n/a	buffer = PyMem_NEW(wchar_t, buflen);
3017	n/a	if (buffer == NULL) {
3018	n/a	PyErr_NoMemory();
3019	n/a	return NULL;
3020	n/a	}
3021	n/a	buflen = unicode_aswidechar(unicode, buffer, buflen);
3022	n/a	if (buflen == -1) {
3023	n/a	PyMem_FREE(buffer);
3024	n/a	return NULL;
3025	n/a	}
3026	n/a	if (size != NULL)
3027	n/a	*size = buflen;
3028	n/a	return buffer;
3029	n/a	}
3030	n/a
3031	n/a	#endif /* HAVE_WCHAR_H */
3032	n/a
3033	n/a	PyObject *
3034	n/a	PyUnicode_FromOrdinal(int ordinal)
3035	n/a	{
3036	n/a	if (ordinal < 0 \|\| ordinal > MAX_UNICODE) {
3037	n/a	PyErr_SetString(PyExc_ValueError,
3038	n/a	"chr() arg not in range(0x110000)");
3039	n/a	return NULL;
3040	n/a	}
3041	n/a
3042	n/a	return unicode_char((Py_UCS4)ordinal);
3043	n/a	}
3044	n/a
3045	n/a	PyObject *
3046	n/a	PyUnicode_FromObject(PyObject *obj)
3047	n/a	{
3048	n/a	/* XXX Perhaps we should make this API an alias of
3049	n/a	PyObject_Str() instead ?! */
3050	n/a	if (PyUnicode_CheckExact(obj)) {
3051	n/a	if (PyUnicode_READY(obj) == -1)
3052	n/a	return NULL;
3053	n/a	Py_INCREF(obj);
3054	n/a	return obj;
3055	n/a	}
3056	n/a	if (PyUnicode_Check(obj)) {
3057	n/a	/* For a Unicode subtype that's not a Unicode object,
3058	n/a	return a true Unicode object with the same data. */
3059	n/a	return _PyUnicode_Copy(obj);
3060	n/a	}
3061	n/a	PyErr_Format(PyExc_TypeError,
3062	n/a	"Can't convert '%.100s' object to str implicitly",
3063	n/a	Py_TYPE(obj)->tp_name);
3064	n/a	return NULL;
3065	n/a	}
3066	n/a
3067	n/a	PyObject *
3068	n/a	PyUnicode_FromEncodedObject(PyObject *obj,
3069	n/a	const char *encoding,
3070	n/a	const char *errors)
3071	n/a	{
3072	n/a	Py_buffer buffer;
3073	n/a	PyObject *v;
3074	n/a
3075	n/a	if (obj == NULL) {
3076	n/a	PyErr_BadInternalCall();
3077	n/a	return NULL;
3078	n/a	}
3079	n/a
3080	n/a	/* Decoding bytes objects is the most common case and should be fast */
3081	n/a	if (PyBytes_Check(obj)) {
3082	n/a	if (PyBytes_GET_SIZE(obj) == 0)
3083	n/a	_Py_RETURN_UNICODE_EMPTY();
3084	n/a	v = PyUnicode_Decode(
3085	n/a	PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj),
3086	n/a	encoding, errors);
3087	n/a	return v;
3088	n/a	}
3089	n/a
3090	n/a	if (PyUnicode_Check(obj)) {
3091	n/a	PyErr_SetString(PyExc_TypeError,
3092	n/a	"decoding str is not supported");
3093	n/a	return NULL;
3094	n/a	}
3095	n/a
3096	n/a	/* Retrieve a bytes buffer view through the PEP 3118 buffer interface */
3097	n/a	if (PyObject_GetBuffer(obj, &buffer, PyBUF_SIMPLE) < 0) {
3098	n/a	PyErr_Format(PyExc_TypeError,
3099	n/a	"decoding to str: need a bytes-like object, %.80s found",
3100	n/a	Py_TYPE(obj)->tp_name);
3101	n/a	return NULL;
3102	n/a	}
3103	n/a
3104	n/a	if (buffer.len == 0) {
3105	n/a	PyBuffer_Release(&buffer);
3106	n/a	_Py_RETURN_UNICODE_EMPTY();
3107	n/a	}
3108	n/a
3109	n/a	v = PyUnicode_Decode((char*) buffer.buf, buffer.len, encoding, errors);
3110	n/a	PyBuffer_Release(&buffer);
3111	n/a	return v;
3112	n/a	}
3113	n/a
3114	n/a	/* Normalize an encoding name: similar to encodings.normalize_encoding(), but
3115	n/a	also convert to lowercase. Return 1 on success, or 0 on error (encoding is
3116	n/a	longer than lower_len-1). */
3117	n/a	int
3118	n/a	_Py_normalize_encoding(const char *encoding,
3119	n/a	char *lower,
3120	n/a	size_t lower_len)
3121	n/a	{
3122	n/a	const char *e;
3123	n/a	char *l;
3124	n/a	char *l_end;
3125	n/a	int punct;
3126	n/a
3127	n/a	assert(encoding != NULL);
3128	n/a
3129	n/a	e = encoding;
3130	n/a	l = lower;
3131	n/a	l_end = &lower[lower_len - 1];
3132	n/a	punct = 0;
3133	n/a	while (1) {
3134	n/a	char c = *e;
3135	n/a	if (c == 0) {
3136	n/a	break;
3137	n/a	}
3138	n/a
3139	n/a	if (Py_ISALNUM(c) \|\| c == '.') {
3140	n/a	if (punct && l != lower) {
3141	n/a	if (l == l_end) {
3142	n/a	return 0;
3143	n/a	}
3144	n/a	*l++ = '_';
3145	n/a	}
3146	n/a	punct = 0;
3147	n/a
3148	n/a	if (l == l_end) {
3149	n/a	return 0;
3150	n/a	}
3151	n/a	*l++ = Py_TOLOWER(c);
3152	n/a	}
3153	n/a	else {
3154	n/a	punct = 1;
3155	n/a	}
3156	n/a
3157	n/a	e++;
3158	n/a	}
3159	n/a	*l = '\0';
3160	n/a	return 1;
3161	n/a	}
3162	n/a
3163	n/a	PyObject *
3164	n/a	PyUnicode_Decode(const char *s,
3165	n/a	Py_ssize_t size,
3166	n/a	const char *encoding,
3167	n/a	const char *errors)
3168	n/a	{
3169	n/a	PyObject buffer = NULL, unicode;
3170	n/a	Py_buffer info;
3171	n/a	char buflower[11]; /* strlen("iso-8859-1\0") == 11, longest shortcut */
3172	n/a
3173	n/a	if (encoding == NULL) {
3174	n/a	return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL);
3175	n/a	}
3176	n/a
3177	n/a	/* Shortcuts for common default encodings */
3178	n/a	if (_Py_normalize_encoding(encoding, buflower, sizeof(buflower))) {
3179	n/a	char *lower = buflower;
3180	n/a
3181	n/a	/* Fast paths */
3182	n/a	if (lower[0] == 'u' && lower[1] == 't' && lower[2] == 'f') {
3183	n/a	lower += 3;
3184	n/a	if (*lower == '_') {
3185	n/a	/* Match "utf8" and "utf_8" */
3186	n/a	lower++;
3187	n/a	}
3188	n/a
3189	n/a	if (lower[0] == '8' && lower[1] == 0) {
3190	n/a	return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL);
3191	n/a	}
3192	n/a	else if (lower[0] == '1' && lower[1] == '6' && lower[2] == 0) {
3193	n/a	return PyUnicode_DecodeUTF16(s, size, errors, 0);
3194	n/a	}
3195	n/a	else if (lower[0] == '3' && lower[1] == '2' && lower[2] == 0) {
3196	n/a	return PyUnicode_DecodeUTF32(s, size, errors, 0);
3197	n/a	}
3198	n/a	}
3199	n/a	else {
3200	n/a	if (strcmp(lower, "ascii") == 0
3201	n/a	\|\| strcmp(lower, "us_ascii") == 0) {
3202	n/a	return PyUnicode_DecodeASCII(s, size, errors);
3203	n/a	}
3204	n/a	#ifdef MS_WINDOWS
3205	n/a	else if (strcmp(lower, "mbcs") == 0) {
3206	n/a	return PyUnicode_DecodeMBCS(s, size, errors);
3207	n/a	}
3208	n/a	#endif
3209	n/a	else if (strcmp(lower, "latin1") == 0
3210	n/a	\|\| strcmp(lower, "latin_1") == 0
3211	n/a	\|\| strcmp(lower, "iso_8859_1") == 0
3212	n/a	\|\| strcmp(lower, "iso8859_1") == 0) {
3213	n/a	return PyUnicode_DecodeLatin1(s, size, errors);
3214	n/a	}
3215	n/a	}
3216	n/a	}
3217	n/a
3218	n/a	/* Decode via the codec registry */
3219	n/a	buffer = NULL;
3220	n/a	if (PyBuffer_FillInfo(&info, NULL, (void *)s, size, 1, PyBUF_FULL_RO) < 0)
3221	n/a	goto onError;
3222	n/a	buffer = PyMemoryView_FromBuffer(&info);
3223	n/a	if (buffer == NULL)
3224	n/a	goto onError;
3225	n/a	unicode = _PyCodec_DecodeText(buffer, encoding, errors);
3226	n/a	if (unicode == NULL)
3227	n/a	goto onError;
3228	n/a	if (!PyUnicode_Check(unicode)) {
3229	n/a	PyErr_Format(PyExc_TypeError,
3230	n/a	"'%.400s' decoder returned '%.400s' instead of 'str'; "
3231	n/a	"use codecs.decode() to decode to arbitrary types",
3232	n/a	encoding,
3233	n/a	Py_TYPE(unicode)->tp_name);
3234	n/a	Py_DECREF(unicode);
3235	n/a	goto onError;
3236	n/a	}
3237	n/a	Py_DECREF(buffer);
3238	n/a	return unicode_result(unicode);
3239	n/a
3240	n/a	onError:
3241	n/a	Py_XDECREF(buffer);
3242	n/a	return NULL;
3243	n/a	}
3244	n/a
3245	n/a	PyObject *
3246	n/a	PyUnicode_AsDecodedObject(PyObject *unicode,
3247	n/a	const char *encoding,
3248	n/a	const char *errors)
3249	n/a	{
3250	n/a	if (!PyUnicode_Check(unicode)) {
3251	n/a	PyErr_BadArgument();
3252	n/a	return NULL;
3253	n/a	}
3254	n/a
3255	n/a	if (PyErr_WarnEx(PyExc_DeprecationWarning,
3256	n/a	"PyUnicode_AsDecodedObject() is deprecated; "
3257	n/a	"use PyCodec_Decode() to decode from str", 1) < 0)
3258	n/a	return NULL;
3259	n/a
3260	n/a	if (encoding == NULL)
3261	n/a	encoding = PyUnicode_GetDefaultEncoding();
3262	n/a
3263	n/a	/* Decode via the codec registry */
3264	n/a	return PyCodec_Decode(unicode, encoding, errors);
3265	n/a	}
3266	n/a
3267	n/a	PyObject *
3268	n/a	PyUnicode_AsDecodedUnicode(PyObject *unicode,
3269	n/a	const char *encoding,
3270	n/a	const char *errors)
3271	n/a	{
3272	n/a	PyObject *v;
3273	n/a
3274	n/a	if (!PyUnicode_Check(unicode)) {
3275	n/a	PyErr_BadArgument();
3276	n/a	goto onError;
3277	n/a	}
3278	n/a
3279	n/a	if (PyErr_WarnEx(PyExc_DeprecationWarning,
3280	n/a	"PyUnicode_AsDecodedUnicode() is deprecated; "
3281	n/a	"use PyCodec_Decode() to decode from str to str", 1) < 0)
3282	n/a	return NULL;
3283	n/a
3284	n/a	if (encoding == NULL)
3285	n/a	encoding = PyUnicode_GetDefaultEncoding();
3286	n/a
3287	n/a	/* Decode via the codec registry */
3288	n/a	v = PyCodec_Decode(unicode, encoding, errors);
3289	n/a	if (v == NULL)
3290	n/a	goto onError;
3291	n/a	if (!PyUnicode_Check(v)) {
3292	n/a	PyErr_Format(PyExc_TypeError,
3293	n/a	"'%.400s' decoder returned '%.400s' instead of 'str'; "
3294	n/a	"use codecs.decode() to decode to arbitrary types",
3295	n/a	encoding,
3296	n/a	Py_TYPE(unicode)->tp_name);
3297	n/a	Py_DECREF(v);
3298	n/a	goto onError;
3299	n/a	}
3300	n/a	return unicode_result(v);
3301	n/a
3302	n/a	onError:
3303	n/a	return NULL;
3304	n/a	}
3305	n/a
3306	n/a	PyObject *
3307	n/a	PyUnicode_Encode(const Py_UNICODE *s,
3308	n/a	Py_ssize_t size,
3309	n/a	const char *encoding,
3310	n/a	const char *errors)
3311	n/a	{
3312	n/a	PyObject v, unicode;
3313	n/a
3314	n/a	unicode = PyUnicode_FromWideChar(s, size);
3315	n/a	if (unicode == NULL)
3316	n/a	return NULL;
3317	n/a	v = PyUnicode_AsEncodedString(unicode, encoding, errors);
3318	n/a	Py_DECREF(unicode);
3319	n/a	return v;
3320	n/a	}
3321	n/a
3322	n/a	PyObject *
3323	n/a	PyUnicode_AsEncodedObject(PyObject *unicode,
3324	n/a	const char *encoding,
3325	n/a	const char *errors)
3326	n/a	{
3327	n/a	PyObject *v;
3328	n/a
3329	n/a	if (!PyUnicode_Check(unicode)) {
3330	n/a	PyErr_BadArgument();
3331	n/a	goto onError;
3332	n/a	}
3333	n/a
3334	n/a	if (PyErr_WarnEx(PyExc_DeprecationWarning,
3335	n/a	"PyUnicode_AsEncodedObject() is deprecated; "
3336	n/a	"use PyUnicode_AsEncodedString() to encode from str to bytes "
3337	n/a	"or PyCodec_Encode() for generic encoding", 1) < 0)
3338	n/a	return NULL;
3339	n/a
3340	n/a	if (encoding == NULL)
3341	n/a	encoding = PyUnicode_GetDefaultEncoding();
3342	n/a
3343	n/a	/* Encode via the codec registry */
3344	n/a	v = PyCodec_Encode(unicode, encoding, errors);
3345	n/a	if (v == NULL)
3346	n/a	goto onError;
3347	n/a	return v;
3348	n/a
3349	n/a	onError:
3350	n/a	return NULL;
3351	n/a	}
3352	n/a
3353	n/a	static size_t
3354	n/a	wcstombs_errorpos(const wchar_t *wstr)
3355	n/a	{
3356	n/a	size_t len;
3357	n/a	#if SIZEOF_WCHAR_T == 2
3358	n/a	wchar_t buf[3];
3359	n/a	#else
3360	n/a	wchar_t buf[2];
3361	n/a	#endif
3362	n/a	char outbuf[MB_LEN_MAX];
3363	n/a	const wchar_t start, previous;
3364	n/a
3365	n/a	#if SIZEOF_WCHAR_T == 2
3366	n/a	buf[2] = 0;
3367	n/a	#else
3368	n/a	buf[1] = 0;
3369	n/a	#endif
3370	n/a	start = wstr;
3371	n/a	while (*wstr != L'\0')
3372	n/a	{
3373	n/a	previous = wstr;
3374	n/a	#if SIZEOF_WCHAR_T == 2
3375	n/a	if (Py_UNICODE_IS_HIGH_SURROGATE(wstr[0])
3376	n/a	&& Py_UNICODE_IS_LOW_SURROGATE(wstr[1]))
3377	n/a	{
3378	n/a	buf[0] = wstr[0];
3379	n/a	buf[1] = wstr[1];
3380	n/a	wstr += 2;
3381	n/a	}
3382	n/a	else {
3383	n/a	buf[0] = *wstr;
3384	n/a	buf[1] = 0;
3385	n/a	wstr++;
3386	n/a	}
3387	n/a	#else
3388	n/a	buf[0] = *wstr;
3389	n/a	wstr++;
3390	n/a	#endif
3391	n/a	len = wcstombs(outbuf, buf, sizeof(outbuf));
3392	n/a	if (len == (size_t)-1)
3393	n/a	return previous - start;
3394	n/a	}
3395	n/a
3396	n/a	/* failed to find the unencodable character */
3397	n/a	return 0;
3398	n/a	}
3399	n/a
3400	n/a	static int
3401	n/a	locale_error_handler(const char errors, int surrogateescape)
3402	n/a	{
3403	n/a	_Py_error_handler error_handler = get_error_handler(errors);
3404	n/a	switch (error_handler)
3405	n/a	{
3406	n/a	case _Py_ERROR_STRICT:
3407	n/a	*surrogateescape = 0;
3408	n/a	return 0;
3409	n/a	case _Py_ERROR_SURROGATEESCAPE:
3410	n/a	*surrogateescape = 1;
3411	n/a	return 0;
3412	n/a	default:
3413	n/a	PyErr_Format(PyExc_ValueError,
3414	n/a	"only 'strict' and 'surrogateescape' error handlers "
3415	n/a	"are supported, not '%s'",
3416	n/a	errors);
3417	n/a	return -1;
3418	n/a	}
3419	n/a	}
3420	n/a
3421	n/a	PyObject *
3422	n/a	PyUnicode_EncodeLocale(PyObject unicode, const char errors)
3423	n/a	{
3424	n/a	Py_ssize_t wlen, wlen2;
3425	n/a	wchar_t *wstr;
3426	n/a	char *errmsg;
3427	n/a	PyObject bytes, reason, *exc;
3428	n/a	size_t error_pos, errlen;
3429	n/a	int surrogateescape;
3430	n/a
3431	n/a	if (locale_error_handler(errors, &surrogateescape) < 0)
3432	n/a	return NULL;
3433	n/a
3434	n/a	wstr = PyUnicode_AsWideCharString(unicode, &wlen);
3435	n/a	if (wstr == NULL)
3436	n/a	return NULL;
3437	n/a
3438	n/a	wlen2 = wcslen(wstr);
3439	n/a	if (wlen2 != wlen) {
3440	n/a	PyMem_Free(wstr);
3441	n/a	PyErr_SetString(PyExc_ValueError, "embedded null character");
3442	n/a	return NULL;
3443	n/a	}
3444	n/a
3445	n/a	if (surrogateescape) {
3446	n/a	/* "surrogateescape" error handler */
3447	n/a	char *str;
3448	n/a
3449	n/a	str = Py_EncodeLocale(wstr, &error_pos);
3450	n/a	if (str == NULL) {
3451	n/a	if (error_pos == (size_t)-1) {
3452	n/a	PyErr_NoMemory();
3453	n/a	PyMem_Free(wstr);
3454	n/a	return NULL;
3455	n/a	}
3456	n/a	else {
3457	n/a	goto encode_error;
3458	n/a	}
3459	n/a	}
3460	n/a	PyMem_Free(wstr);
3461	n/a
3462	n/a	bytes = PyBytes_FromString(str);
3463	n/a	PyMem_Free(str);
3464	n/a	}
3465	n/a	else {
3466	n/a	/* strict mode */
3467	n/a	size_t len, len2;
3468	n/a
3469	n/a	len = wcstombs(NULL, wstr, 0);
3470	n/a	if (len == (size_t)-1) {
3471	n/a	error_pos = (size_t)-1;
3472	n/a	goto encode_error;
3473	n/a	}
3474	n/a
3475	n/a	bytes = PyBytes_FromStringAndSize(NULL, len);
3476	n/a	if (bytes == NULL) {
3477	n/a	PyMem_Free(wstr);
3478	n/a	return NULL;
3479	n/a	}
3480	n/a
3481	n/a	len2 = wcstombs(PyBytes_AS_STRING(bytes), wstr, len+1);
3482	n/a	if (len2 == (size_t)-1 \|\| len2 > len) {
3483	n/a	Py_DECREF(bytes);
3484	n/a	error_pos = (size_t)-1;
3485	n/a	goto encode_error;
3486	n/a	}
3487	n/a	PyMem_Free(wstr);
3488	n/a	}
3489	n/a	return bytes;
3490	n/a
3491	n/a	encode_error:
3492	n/a	errmsg = strerror(errno);
3493	n/a	assert(errmsg != NULL);
3494	n/a
3495	n/a	if (error_pos == (size_t)-1)
3496	n/a	error_pos = wcstombs_errorpos(wstr);
3497	n/a
3498	n/a	PyMem_Free(wstr);
3499	n/a
3500	n/a	wstr = Py_DecodeLocale(errmsg, &errlen);
3501	n/a	if (wstr != NULL) {
3502	n/a	reason = PyUnicode_FromWideChar(wstr, errlen);
3503	n/a	PyMem_RawFree(wstr);
3504	n/a	} else {
3505	n/a	errmsg = NULL;
3506	n/a	}
3507	n/a
3508	n/a	if (errmsg == NULL)
3509	n/a	reason = PyUnicode_FromString(
3510	n/a	"wcstombs() encountered an unencodable "
3511	n/a	"wide character");
3512	n/a	if (reason == NULL)
3513	n/a	return NULL;
3514	n/a
3515	n/a	exc = PyObject_CallFunction(PyExc_UnicodeEncodeError, "sOnnO",
3516	n/a	"locale", unicode,
3517	n/a	(Py_ssize_t)error_pos,
3518	n/a	(Py_ssize_t)(error_pos+1),
3519	n/a	reason);
3520	n/a	Py_DECREF(reason);
3521	n/a	if (exc != NULL) {
3522	n/a	PyCodec_StrictErrors(exc);
3523	n/a	Py_DECREF(exc);
3524	n/a	}
3525	n/a	return NULL;
3526	n/a	}
3527	n/a
3528	n/a	PyObject *
3529	n/a	PyUnicode_EncodeFSDefault(PyObject *unicode)
3530	n/a	{
3531	n/a	#if defined(__APPLE__)
3532	n/a	return _PyUnicode_AsUTF8String(unicode, Py_FileSystemDefaultEncodeErrors);
3533	n/a	#else
3534	n/a	PyInterpreterState *interp = PyThreadState_GET()->interp;
3535	n/a	/* Bootstrap check: if the filesystem codec is implemented in Python, we
3536	n/a	cannot use it to encode and decode filenames before it is loaded. Load
3537	n/a	the Python codec requires to encode at least its own filename. Use the C
3538	n/a	version of the locale codec until the codec registry is initialized and
3539	n/a	the Python codec is loaded.
3540	n/a
3541	n/a	Py_FileSystemDefaultEncoding is shared between all interpreters, we
3542	n/a	cannot only rely on it: check also interp->fscodec_initialized for
3543	n/a	subinterpreters. */
3544	n/a	if (Py_FileSystemDefaultEncoding && interp->fscodec_initialized) {
3545	n/a	return PyUnicode_AsEncodedString(unicode,
3546	n/a	Py_FileSystemDefaultEncoding,
3547	n/a	Py_FileSystemDefaultEncodeErrors);
3548	n/a	}
3549	n/a	else {
3550	n/a	return PyUnicode_EncodeLocale(unicode, Py_FileSystemDefaultEncodeErrors);
3551	n/a	}
3552	n/a	#endif
3553	n/a	}
3554	n/a
3555	n/a	PyObject *
3556	n/a	PyUnicode_AsEncodedString(PyObject *unicode,
3557	n/a	const char *encoding,
3558	n/a	const char *errors)
3559	n/a	{
3560	n/a	PyObject *v;
3561	n/a	char buflower[11]; /* strlen("iso_8859_1\0") == 11, longest shortcut */
3562	n/a
3563	n/a	if (!PyUnicode_Check(unicode)) {
3564	n/a	PyErr_BadArgument();
3565	n/a	return NULL;
3566	n/a	}
3567	n/a
3568	n/a	if (encoding == NULL) {
3569	n/a	return _PyUnicode_AsUTF8String(unicode, errors);
3570	n/a	}
3571	n/a
3572	n/a	/* Shortcuts for common default encodings */
3573	n/a	if (_Py_normalize_encoding(encoding, buflower, sizeof(buflower))) {
3574	n/a	char *lower = buflower;
3575	n/a
3576	n/a	/* Fast paths */
3577	n/a	if (lower[0] == 'u' && lower[1] == 't' && lower[2] == 'f') {
3578	n/a	lower += 3;
3579	n/a	if (*lower == '_') {
3580	n/a	/* Match "utf8" and "utf_8" */
3581	n/a	lower++;
3582	n/a	}
3583	n/a
3584	n/a	if (lower[0] == '8' && lower[1] == 0) {
3585	n/a	return _PyUnicode_AsUTF8String(unicode, errors);
3586	n/a	}
3587	n/a	else if (lower[0] == '1' && lower[1] == '6' && lower[2] == 0) {
3588	n/a	return _PyUnicode_EncodeUTF16(unicode, errors, 0);
3589	n/a	}
3590	n/a	else if (lower[0] == '3' && lower[1] == '2' && lower[2] == 0) {
3591	n/a	return _PyUnicode_EncodeUTF32(unicode, errors, 0);
3592	n/a	}
3593	n/a	}
3594	n/a	else {
3595	n/a	if (strcmp(lower, "ascii") == 0
3596	n/a	\|\| strcmp(lower, "us_ascii") == 0) {
3597	n/a	return _PyUnicode_AsASCIIString(unicode, errors);
3598	n/a	}
3599	n/a	#ifdef MS_WINDOWS
3600	n/a	else if (strcmp(lower, "mbcs") == 0) {
3601	n/a	return PyUnicode_EncodeCodePage(CP_ACP, unicode, errors);
3602	n/a	}
3603	n/a	#endif
3604	n/a	else if (strcmp(lower, "latin1") == 0 \|\|
3605	n/a	strcmp(lower, "latin_1") == 0 \|\|
3606	n/a	strcmp(lower, "iso_8859_1") == 0 \|\|
3607	n/a	strcmp(lower, "iso8859_1") == 0) {
3608	n/a	return _PyUnicode_AsLatin1String(unicode, errors);
3609	n/a	}
3610	n/a	}
3611	n/a	}
3612	n/a
3613	n/a	/* Encode via the codec registry */
3614	n/a	v = _PyCodec_EncodeText(unicode, encoding, errors);
3615	n/a	if (v == NULL)
3616	n/a	return NULL;
3617	n/a
3618	n/a	/* The normal path */
3619	n/a	if (PyBytes_Check(v))
3620	n/a	return v;
3621	n/a
3622	n/a	/* If the codec returns a buffer, raise a warning and convert to bytes */
3623	n/a	if (PyByteArray_Check(v)) {
3624	n/a	int error;
3625	n/a	PyObject *b;
3626	n/a
3627	n/a	error = PyErr_WarnFormat(PyExc_RuntimeWarning, 1,
3628	n/a	"encoder %s returned bytearray instead of bytes; "
3629	n/a	"use codecs.encode() to encode to arbitrary types",
3630	n/a	encoding);
3631	n/a	if (error) {
3632	n/a	Py_DECREF(v);
3633	n/a	return NULL;
3634	n/a	}
3635	n/a
3636	n/a	b = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(v), Py_SIZE(v));
3637	n/a	Py_DECREF(v);
3638	n/a	return b;
3639	n/a	}
3640	n/a
3641	n/a	PyErr_Format(PyExc_TypeError,
3642	n/a	"'%.400s' encoder returned '%.400s' instead of 'bytes'; "
3643	n/a	"use codecs.encode() to encode to arbitrary types",
3644	n/a	encoding,
3645	n/a	Py_TYPE(v)->tp_name);
3646	n/a	Py_DECREF(v);
3647	n/a	return NULL;
3648	n/a	}
3649	n/a
3650	n/a	PyObject *
3651	n/a	PyUnicode_AsEncodedUnicode(PyObject *unicode,
3652	n/a	const char *encoding,
3653	n/a	const char *errors)
3654	n/a	{
3655	n/a	PyObject *v;
3656	n/a
3657	n/a	if (!PyUnicode_Check(unicode)) {
3658	n/a	PyErr_BadArgument();
3659	n/a	goto onError;
3660	n/a	}
3661	n/a
3662	n/a	if (PyErr_WarnEx(PyExc_DeprecationWarning,
3663	n/a	"PyUnicode_AsEncodedUnicode() is deprecated; "
3664	n/a	"use PyCodec_Encode() to encode from str to str", 1) < 0)
3665	n/a	return NULL;
3666	n/a
3667	n/a	if (encoding == NULL)
3668	n/a	encoding = PyUnicode_GetDefaultEncoding();
3669	n/a
3670	n/a	/* Encode via the codec registry */
3671	n/a	v = PyCodec_Encode(unicode, encoding, errors);
3672	n/a	if (v == NULL)
3673	n/a	goto onError;
3674	n/a	if (!PyUnicode_Check(v)) {
3675	n/a	PyErr_Format(PyExc_TypeError,
3676	n/a	"'%.400s' encoder returned '%.400s' instead of 'str'; "
3677	n/a	"use codecs.encode() to encode to arbitrary types",
3678	n/a	encoding,
3679	n/a	Py_TYPE(v)->tp_name);
3680	n/a	Py_DECREF(v);
3681	n/a	goto onError;
3682	n/a	}
3683	n/a	return v;
3684	n/a
3685	n/a	onError:
3686	n/a	return NULL;
3687	n/a	}
3688	n/a
3689	n/a	static size_t
3690	n/a	mbstowcs_errorpos(const char *str, size_t len)
3691	n/a	{
3692	n/a	#ifdef HAVE_MBRTOWC
3693	n/a	const char *start = str;
3694	n/a	mbstate_t mbs;
3695	n/a	size_t converted;
3696	n/a	wchar_t ch;
3697	n/a
3698	n/a	memset(&mbs, 0, sizeof mbs);
3699	n/a	while (len)
3700	n/a	{
3701	n/a	converted = mbrtowc(&ch, str, len, &mbs);
3702	n/a	if (converted == 0)
3703	n/a	/* Reached end of string */
3704	n/a	break;
3705	n/a	if (converted == (size_t)-1 \|\| converted == (size_t)-2) {
3706	n/a	/* Conversion error or incomplete character */
3707	n/a	return str - start;
3708	n/a	}
3709	n/a	else {
3710	n/a	str += converted;
3711	n/a	len -= converted;
3712	n/a	}
3713	n/a	}
3714	n/a	/* failed to find the undecodable byte sequence */
3715	n/a	return 0;
3716	n/a	#endif
3717	n/a	return 0;
3718	n/a	}
3719	n/a
3720	n/a	PyObject*
3721	n/a	PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len,
3722	n/a	const char *errors)
3723	n/a	{
3724	n/a	wchar_t smallbuf[256];
3725	n/a	size_t smallbuf_len = Py_ARRAY_LENGTH(smallbuf);
3726	n/a	wchar_t *wstr;
3727	n/a	size_t wlen, wlen2;
3728	n/a	PyObject *unicode;
3729	n/a	int surrogateescape;
3730	n/a	size_t error_pos, errlen;
3731	n/a	char *errmsg;
3732	n/a	PyObject exc, reason = NULL; /* initialize to prevent gcc warning */
3733	n/a
3734	n/a	if (locale_error_handler(errors, &surrogateescape) < 0)
3735	n/a	return NULL;
3736	n/a
3737	n/a	if (str[len] != '\0' \|\| (size_t)len != strlen(str)) {
3738	n/a	PyErr_SetString(PyExc_ValueError, "embedded null byte");
3739	n/a	return NULL;
3740	n/a	}
3741	n/a
3742	n/a	if (surrogateescape) {
3743	n/a	/* "surrogateescape" error handler */
3744	n/a	wstr = Py_DecodeLocale(str, &wlen);
3745	n/a	if (wstr == NULL) {
3746	n/a	if (wlen == (size_t)-1)
3747	n/a	PyErr_NoMemory();
3748	n/a	else
3749	n/a	PyErr_SetFromErrno(PyExc_OSError);
3750	n/a	return NULL;
3751	n/a	}
3752	n/a
3753	n/a	unicode = PyUnicode_FromWideChar(wstr, wlen);
3754	n/a	PyMem_RawFree(wstr);
3755	n/a	}
3756	n/a	else {
3757	n/a	/* strict mode */
3758	n/a	#ifndef HAVE_BROKEN_MBSTOWCS
3759	n/a	wlen = mbstowcs(NULL, str, 0);
3760	n/a	#else
3761	n/a	wlen = len;
3762	n/a	#endif
3763	n/a	if (wlen == (size_t)-1)
3764	n/a	goto decode_error;
3765	n/a	if (wlen+1 <= smallbuf_len) {
3766	n/a	wstr = smallbuf;
3767	n/a	}
3768	n/a	else {
3769	n/a	wstr = PyMem_New(wchar_t, wlen+1);
3770	n/a	if (!wstr)
3771	n/a	return PyErr_NoMemory();
3772	n/a	}
3773	n/a
3774	n/a	wlen2 = mbstowcs(wstr, str, wlen+1);
3775	n/a	if (wlen2 == (size_t)-1) {
3776	n/a	if (wstr != smallbuf)
3777	n/a	PyMem_Free(wstr);
3778	n/a	goto decode_error;
3779	n/a	}
3780	n/a	#ifdef HAVE_BROKEN_MBSTOWCS
3781	n/a	assert(wlen2 == wlen);
3782	n/a	#endif
3783	n/a	unicode = PyUnicode_FromWideChar(wstr, wlen2);
3784	n/a	if (wstr != smallbuf)
3785	n/a	PyMem_Free(wstr);
3786	n/a	}
3787	n/a	return unicode;
3788	n/a
3789	n/a	decode_error:
3790	n/a	errmsg = strerror(errno);
3791	n/a	assert(errmsg != NULL);
3792	n/a
3793	n/a	error_pos = mbstowcs_errorpos(str, len);
3794	n/a	wstr = Py_DecodeLocale(errmsg, &errlen);
3795	n/a	if (wstr != NULL) {
3796	n/a	reason = PyUnicode_FromWideChar(wstr, errlen);
3797	n/a	PyMem_RawFree(wstr);
3798	n/a	}
3799	n/a
3800	n/a	if (reason == NULL)
3801	n/a	reason = PyUnicode_FromString(
3802	n/a	"mbstowcs() encountered an invalid multibyte sequence");
3803	n/a	if (reason == NULL)
3804	n/a	return NULL;
3805	n/a
3806	n/a	exc = PyObject_CallFunction(PyExc_UnicodeDecodeError, "sy#nnO",
3807	n/a	"locale", str, len,
3808	n/a	(Py_ssize_t)error_pos,
3809	n/a	(Py_ssize_t)(error_pos+1),
3810	n/a	reason);
3811	n/a	Py_DECREF(reason);
3812	n/a	if (exc != NULL) {
3813	n/a	PyCodec_StrictErrors(exc);
3814	n/a	Py_DECREF(exc);
3815	n/a	}
3816	n/a	return NULL;
3817	n/a	}
3818	n/a
3819	n/a	PyObject*
3820	n/a	PyUnicode_DecodeLocale(const char str, const char errors)
3821	n/a	{
3822	n/a	Py_ssize_t size = (Py_ssize_t)strlen(str);
3823	n/a	return PyUnicode_DecodeLocaleAndSize(str, size, errors);
3824	n/a	}
3825	n/a
3826	n/a
3827	n/a	PyObject*
3828	n/a	PyUnicode_DecodeFSDefault(const char *s) {
3829	n/a	Py_ssize_t size = (Py_ssize_t)strlen(s);
3830	n/a	return PyUnicode_DecodeFSDefaultAndSize(s, size);
3831	n/a	}
3832	n/a
3833	n/a	PyObject*
3834	n/a	PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size)
3835	n/a	{
3836	n/a	#if defined(__APPLE__)
3837	n/a	return PyUnicode_DecodeUTF8Stateful(s, size, Py_FileSystemDefaultEncodeErrors, NULL);
3838	n/a	#else
3839	n/a	PyInterpreterState *interp = PyThreadState_GET()->interp;
3840	n/a	/* Bootstrap check: if the filesystem codec is implemented in Python, we
3841	n/a	cannot use it to encode and decode filenames before it is loaded. Load
3842	n/a	the Python codec requires to encode at least its own filename. Use the C
3843	n/a	version of the locale codec until the codec registry is initialized and
3844	n/a	the Python codec is loaded.
3845	n/a
3846	n/a	Py_FileSystemDefaultEncoding is shared between all interpreters, we
3847	n/a	cannot only rely on it: check also interp->fscodec_initialized for
3848	n/a	subinterpreters. */
3849	n/a	if (Py_FileSystemDefaultEncoding && interp->fscodec_initialized) {
3850	n/a	return PyUnicode_Decode(s, size,
3851	n/a	Py_FileSystemDefaultEncoding,
3852	n/a	Py_FileSystemDefaultEncodeErrors);
3853	n/a	}
3854	n/a	else {
3855	n/a	return PyUnicode_DecodeLocaleAndSize(s, size, Py_FileSystemDefaultEncodeErrors);
3856	n/a	}
3857	n/a	#endif
3858	n/a	}
3859	n/a
3860	n/a
3861	n/a	int
3862	n/a	PyUnicode_FSConverter(PyObject* arg, void* addr)
3863	n/a	{
3864	n/a	PyObject *path = NULL;
3865	n/a	PyObject *output = NULL;
3866	n/a	Py_ssize_t size;
3867	n/a	void *data;
3868	n/a	if (arg == NULL) {
3869	n/a	Py_DECREF((PyObject*)addr);
3870	n/a	(PyObject*)addr = NULL;
3871	n/a	return 1;
3872	n/a	}
3873	n/a	path = PyOS_FSPath(arg);
3874	n/a	if (path == NULL) {
3875	n/a	return 0;
3876	n/a	}
3877	n/a	if (PyBytes_Check(path)) {
3878	n/a	output = path;
3879	n/a	}
3880	n/a	else { // PyOS_FSPath() guarantees its returned value is bytes or str.
3881	n/a	output = PyUnicode_EncodeFSDefault(path);
3882	n/a	Py_DECREF(path);
3883	n/a	if (!output) {
3884	n/a	return 0;
3885	n/a	}
3886	n/a	assert(PyBytes_Check(output));
3887	n/a	}
3888	n/a
3889	n/a	size = PyBytes_GET_SIZE(output);
3890	n/a	data = PyBytes_AS_STRING(output);
3891	n/a	if ((size_t)size != strlen(data)) {
3892	n/a	PyErr_SetString(PyExc_ValueError, "embedded null byte");
3893	n/a	Py_DECREF(output);
3894	n/a	return 0;
3895	n/a	}
3896	n/a	(PyObject*)addr = output;
3897	n/a	return Py_CLEANUP_SUPPORTED;
3898	n/a	}
3899	n/a
3900	n/a
3901	n/a	int
3902	n/a	PyUnicode_FSDecoder(PyObject* arg, void* addr)
3903	n/a	{
3904	n/a	int is_buffer = 0;
3905	n/a	PyObject *path = NULL;
3906	n/a	PyObject *output = NULL;
3907	n/a	if (arg == NULL) {
3908	n/a	Py_DECREF((PyObject*)addr);
3909	n/a	return 1;
3910	n/a	}
3911	n/a
3912	n/a	is_buffer = PyObject_CheckBuffer(arg);
3913	n/a	if (!is_buffer) {
3914	n/a	path = PyOS_FSPath(arg);
3915	n/a	if (path == NULL) {
3916	n/a	return 0;
3917	n/a	}
3918	n/a	}
3919	n/a	else {
3920	n/a	path = arg;
3921	n/a	Py_INCREF(arg);
3922	n/a	}
3923	n/a
3924	n/a	if (PyUnicode_Check(path)) {
3925	n/a	if (PyUnicode_READY(path) == -1) {
3926	n/a	Py_DECREF(path);
3927	n/a	return 0;
3928	n/a	}
3929	n/a	output = path;
3930	n/a	}
3931	n/a	else if (PyBytes_Check(path) \|\| is_buffer) {
3932	n/a	PyObject *path_bytes = NULL;
3933	n/a
3934	n/a	if (!PyBytes_Check(path) &&
3935	n/a	PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
3936	n/a	"path should be string, bytes, or os.PathLike, not %.200s",
3937	n/a	Py_TYPE(arg)->tp_name)) {
3938	n/a	Py_DECREF(path);
3939	n/a	return 0;
3940	n/a	}
3941	n/a	path_bytes = PyBytes_FromObject(path);
3942	n/a	Py_DECREF(path);
3943	n/a	if (!path_bytes) {
3944	n/a	return 0;
3945	n/a	}
3946	n/a	output = PyUnicode_DecodeFSDefaultAndSize(PyBytes_AS_STRING(path_bytes),
3947	n/a	PyBytes_GET_SIZE(path_bytes));
3948	n/a	Py_DECREF(path_bytes);
3949	n/a	if (!output) {
3950	n/a	return 0;
3951	n/a	}
3952	n/a	}
3953	n/a	else {
3954	n/a	PyErr_Format(PyExc_TypeError,
3955	n/a	"path should be string, bytes, or os.PathLike, not %.200s",
3956	n/a	Py_TYPE(arg)->tp_name);
3957	n/a	Py_DECREF(path);
3958	n/a	return 0;
3959	n/a	}
3960	n/a	if (PyUnicode_READY(output) == -1) {
3961	n/a	Py_DECREF(output);
3962	n/a	return 0;
3963	n/a	}
3964	n/a	if (findchar(PyUnicode_DATA(output), PyUnicode_KIND(output),
3965	n/a	PyUnicode_GET_LENGTH(output), 0, 1) >= 0) {
3966	n/a	PyErr_SetString(PyExc_ValueError, "embedded null character");
3967	n/a	Py_DECREF(output);
3968	n/a	return 0;
3969	n/a	}
3970	n/a	(PyObject*)addr = output;
3971	n/a	return Py_CLEANUP_SUPPORTED;
3972	n/a	}
3973	n/a
3974	n/a
3975	n/a	const char *
3976	n/a	PyUnicode_AsUTF8AndSize(PyObject unicode, Py_ssize_t psize)
3977	n/a	{
3978	n/a	PyObject *bytes;
3979	n/a
3980	n/a	if (!PyUnicode_Check(unicode)) {
3981	n/a	PyErr_BadArgument();
3982	n/a	return NULL;
3983	n/a	}
3984	n/a	if (PyUnicode_READY(unicode) == -1)
3985	n/a	return NULL;
3986	n/a
3987	n/a	if (PyUnicode_UTF8(unicode) == NULL) {
3988	n/a	assert(!PyUnicode_IS_COMPACT_ASCII(unicode));
3989	n/a	bytes = _PyUnicode_AsUTF8String(unicode, NULL);
3990	n/a	if (bytes == NULL)
3991	n/a	return NULL;
3992	n/a	_PyUnicode_UTF8(unicode) = PyObject_MALLOC(PyBytes_GET_SIZE(bytes) + 1);
3993	n/a	if (_PyUnicode_UTF8(unicode) == NULL) {
3994	n/a	PyErr_NoMemory();
3995	n/a	Py_DECREF(bytes);
3996	n/a	return NULL;
3997	n/a	}
3998	n/a	_PyUnicode_UTF8_LENGTH(unicode) = PyBytes_GET_SIZE(bytes);
3999	n/a	memcpy(_PyUnicode_UTF8(unicode),
4000	n/a	PyBytes_AS_STRING(bytes),
4001	n/a	_PyUnicode_UTF8_LENGTH(unicode) + 1);
4002	n/a	Py_DECREF(bytes);
4003	n/a	}
4004	n/a
4005	n/a	if (psize)
4006	n/a	*psize = PyUnicode_UTF8_LENGTH(unicode);
4007	n/a	return PyUnicode_UTF8(unicode);
4008	n/a	}
4009	n/a
4010	n/a	const char *
4011	n/a	PyUnicode_AsUTF8(PyObject *unicode)
4012	n/a	{
4013	n/a	return PyUnicode_AsUTF8AndSize(unicode, NULL);
4014	n/a	}
4015	n/a
4016	n/a	Py_UNICODE *
4017	n/a	PyUnicode_AsUnicodeAndSize(PyObject unicode, Py_ssize_t size)
4018	n/a	{
4019	n/a	const unsigned char *one_byte;
4020	n/a	#if SIZEOF_WCHAR_T == 4
4021	n/a	const Py_UCS2 *two_bytes;
4022	n/a	#else
4023	n/a	const Py_UCS4 *four_bytes;
4024	n/a	const Py_UCS4 *ucs4_end;
4025	n/a	Py_ssize_t num_surrogates;
4026	n/a	#endif
4027	n/a	wchar_t *w;
4028	n/a	wchar_t *wchar_end;
4029	n/a
4030	n/a	if (!PyUnicode_Check(unicode)) {
4031	n/a	PyErr_BadArgument();
4032	n/a	return NULL;
4033	n/a	}
4034	n/a	if (_PyUnicode_WSTR(unicode) == NULL) {
4035	n/a	/* Non-ASCII compact unicode object */
4036	n/a	assert(_PyUnicode_KIND(unicode) != 0);
4037	n/a	assert(PyUnicode_IS_READY(unicode));
4038	n/a
4039	n/a	if (PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND) {
4040	n/a	#if SIZEOF_WCHAR_T == 2
4041	n/a	four_bytes = PyUnicode_4BYTE_DATA(unicode);
4042	n/a	ucs4_end = four_bytes + _PyUnicode_LENGTH(unicode);
4043	n/a	num_surrogates = 0;
4044	n/a
4045	n/a	for (; four_bytes < ucs4_end; ++four_bytes) {
4046	n/a	if (*four_bytes > 0xFFFF)
4047	n/a	++num_surrogates;
4048	n/a	}
4049	n/a
4050	n/a	_PyUnicode_WSTR(unicode) = (wchar_t *) PyObject_MALLOC(
4051	n/a	sizeof(wchar_t) * (_PyUnicode_LENGTH(unicode) + 1 + num_surrogates));
4052	n/a	if (!_PyUnicode_WSTR(unicode)) {
4053	n/a	PyErr_NoMemory();
4054	n/a	return NULL;
4055	n/a	}
4056	n/a	_PyUnicode_WSTR_LENGTH(unicode) = _PyUnicode_LENGTH(unicode) + num_surrogates;
4057	n/a
4058	n/a	w = _PyUnicode_WSTR(unicode);
4059	n/a	wchar_end = w + _PyUnicode_WSTR_LENGTH(unicode);
4060	n/a	four_bytes = PyUnicode_4BYTE_DATA(unicode);
4061	n/a	for (; four_bytes < ucs4_end; ++four_bytes, ++w) {
4062	n/a	if (*four_bytes > 0xFFFF) {
4063	n/a	assert(*four_bytes <= MAX_UNICODE);
4064	n/a	/* encode surrogate pair in this case */
4065	n/a	w++ = Py_UNICODE_HIGH_SURROGATE(four_bytes);
4066	n/a	w = Py_UNICODE_LOW_SURROGATE(four_bytes);
4067	n/a	}
4068	n/a	else
4069	n/a	w = four_bytes;
4070	n/a
4071	n/a	if (w > wchar_end) {
4072	n/a	assert(0 && "Miscalculated string end");
4073	n/a	}
4074	n/a	}
4075	n/a	*w = 0;
4076	n/a	#else
4077	n/a	/* sizeof(wchar_t) == 4 */
4078	n/a	Py_FatalError("Impossible unicode object state, wstr and str "
4079	n/a	"should share memory already.");
4080	n/a	return NULL;
4081	n/a	#endif
4082	n/a	}
4083	n/a	else {
4084	n/a	if ((size_t)_PyUnicode_LENGTH(unicode) >
4085	n/a	PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) {
4086	n/a	PyErr_NoMemory();
4087	n/a	return NULL;
4088	n/a	}
4089	n/a	_PyUnicode_WSTR(unicode) = (wchar_t ) PyObject_MALLOC(sizeof(wchar_t)
4090	n/a	(_PyUnicode_LENGTH(unicode) + 1));
4091	n/a	if (!_PyUnicode_WSTR(unicode)) {
4092	n/a	PyErr_NoMemory();
4093	n/a	return NULL;
4094	n/a	}
4095	n/a	if (!PyUnicode_IS_COMPACT_ASCII(unicode))
4096	n/a	_PyUnicode_WSTR_LENGTH(unicode) = _PyUnicode_LENGTH(unicode);
4097	n/a	w = _PyUnicode_WSTR(unicode);
4098	n/a	wchar_end = w + _PyUnicode_LENGTH(unicode);
4099	n/a
4100	n/a	if (PyUnicode_KIND(unicode) == PyUnicode_1BYTE_KIND) {
4101	n/a	one_byte = PyUnicode_1BYTE_DATA(unicode);
4102	n/a	for (; w < wchar_end; ++one_byte, ++w)
4103	n/a	w = one_byte;
4104	n/a	/* null-terminate the wstr */
4105	n/a	*w = 0;
4106	n/a	}
4107	n/a	else if (PyUnicode_KIND(unicode) == PyUnicode_2BYTE_KIND) {
4108	n/a	#if SIZEOF_WCHAR_T == 4
4109	n/a	two_bytes = PyUnicode_2BYTE_DATA(unicode);
4110	n/a	for (; w < wchar_end; ++two_bytes, ++w)
4111	n/a	w = two_bytes;
4112	n/a	/* null-terminate the wstr */
4113	n/a	*w = 0;
4114	n/a	#else
4115	n/a	/* sizeof(wchar_t) == 2 */
4116	n/a	PyObject_FREE(_PyUnicode_WSTR(unicode));
4117	n/a	_PyUnicode_WSTR(unicode) = NULL;
4118	n/a	Py_FatalError("Impossible unicode object state, wstr "
4119	n/a	"and str should share memory already.");
4120	n/a	return NULL;
4121	n/a	#endif
4122	n/a	}
4123	n/a	else {
4124	n/a	assert(0 && "This should never happen.");
4125	n/a	}
4126	n/a	}
4127	n/a	}
4128	n/a	if (size != NULL)
4129	n/a	*size = PyUnicode_WSTR_LENGTH(unicode);
4130	n/a	return _PyUnicode_WSTR(unicode);
4131	n/a	}
4132	n/a
4133	n/a	Py_UNICODE *
4134	n/a	PyUnicode_AsUnicode(PyObject *unicode)
4135	n/a	{
4136	n/a	return PyUnicode_AsUnicodeAndSize(unicode, NULL);
4137	n/a	}
4138	n/a
4139	n/a
4140	n/a	Py_ssize_t
4141	n/a	PyUnicode_GetSize(PyObject *unicode)
4142	n/a	{
4143	n/a	if (!PyUnicode_Check(unicode)) {
4144	n/a	PyErr_BadArgument();
4145	n/a	goto onError;
4146	n/a	}
4147	n/a	if (_PyUnicode_WSTR(unicode) == NULL) {
4148	n/a	if (PyUnicode_AsUnicode(unicode) == NULL)
4149	n/a	goto onError;
4150	n/a	}
4151	n/a	return PyUnicode_WSTR_LENGTH(unicode);
4152	n/a
4153	n/a	onError:
4154	n/a	return -1;
4155	n/a	}
4156	n/a
4157	n/a	Py_ssize_t
4158	n/a	PyUnicode_GetLength(PyObject *unicode)
4159	n/a	{
4160	n/a	if (!PyUnicode_Check(unicode)) {
4161	n/a	PyErr_BadArgument();
4162	n/a	return -1;
4163	n/a	}
4164	n/a	if (PyUnicode_READY(unicode) == -1)
4165	n/a	return -1;
4166	n/a	return PyUnicode_GET_LENGTH(unicode);
4167	n/a	}
4168	n/a
4169	n/a	Py_UCS4
4170	n/a	PyUnicode_ReadChar(PyObject *unicode, Py_ssize_t index)
4171	n/a	{
4172	n/a	void *data;
4173	n/a	int kind;
4174	n/a
4175	n/a	if (!PyUnicode_Check(unicode) \|\| PyUnicode_READY(unicode) == -1) {
4176	n/a	PyErr_BadArgument();
4177	n/a	return (Py_UCS4)-1;
4178	n/a	}
4179	n/a	if (index < 0 \|\| index >= PyUnicode_GET_LENGTH(unicode)) {
4180	n/a	PyErr_SetString(PyExc_IndexError, "string index out of range");
4181	n/a	return (Py_UCS4)-1;
4182	n/a	}
4183	n/a	data = PyUnicode_DATA(unicode);
4184	n/a	kind = PyUnicode_KIND(unicode);
4185	n/a	return PyUnicode_READ(kind, data, index);
4186	n/a	}
4187	n/a
4188	n/a	int
4189	n/a	PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, Py_UCS4 ch)
4190	n/a	{
4191	n/a	if (!PyUnicode_Check(unicode) \|\| !PyUnicode_IS_COMPACT(unicode)) {
4192	n/a	PyErr_BadArgument();
4193	n/a	return -1;
4194	n/a	}
4195	n/a	assert(PyUnicode_IS_READY(unicode));
4196	n/a	if (index < 0 \|\| index >= PyUnicode_GET_LENGTH(unicode)) {
4197	n/a	PyErr_SetString(PyExc_IndexError, "string index out of range");
4198	n/a	return -1;
4199	n/a	}
4200	n/a	if (unicode_check_modifiable(unicode))
4201	n/a	return -1;
4202	n/a	if (ch > PyUnicode_MAX_CHAR_VALUE(unicode)) {
4203	n/a	PyErr_SetString(PyExc_ValueError, "character out of range");
4204	n/a	return -1;
4205	n/a	}
4206	n/a	PyUnicode_WRITE(PyUnicode_KIND(unicode), PyUnicode_DATA(unicode),
4207	n/a	index, ch);
4208	n/a	return 0;
4209	n/a	}
4210	n/a
4211	n/a	const char *
4212	n/a	PyUnicode_GetDefaultEncoding(void)
4213	n/a	{
4214	n/a	return "utf-8";
4215	n/a	}
4216	n/a
4217	n/a	/* create or adjust a UnicodeDecodeError */
4218	n/a	static void
4219	n/a	make_decode_exception(PyObject **exceptionObject,
4220	n/a	const char *encoding,
4221	n/a	const char *input, Py_ssize_t length,
4222	n/a	Py_ssize_t startpos, Py_ssize_t endpos,
4223	n/a	const char *reason)
4224	n/a	{
4225	n/a	if (*exceptionObject == NULL) {
4226	n/a	*exceptionObject = PyUnicodeDecodeError_Create(
4227	n/a	encoding, input, length, startpos, endpos, reason);
4228	n/a	}
4229	n/a	else {
4230	n/a	if (PyUnicodeDecodeError_SetStart(*exceptionObject, startpos))
4231	n/a	goto onError;
4232	n/a	if (PyUnicodeDecodeError_SetEnd(*exceptionObject, endpos))
4233	n/a	goto onError;
4234	n/a	if (PyUnicodeDecodeError_SetReason(*exceptionObject, reason))
4235	n/a	goto onError;
4236	n/a	}
4237	n/a	return;
4238	n/a
4239	n/a	onError:
4240	n/a	Py_CLEAR(*exceptionObject);
4241	n/a	}
4242	n/a
4243	n/a	#ifdef MS_WINDOWS
4244	n/a	/* error handling callback helper:
4245	n/a	build arguments, call the callback and check the arguments,
4246	n/a	if no exception occurred, copy the replacement to the output
4247	n/a	and adjust various state variables.
4248	n/a	return 0 on success, -1 on error
4249	n/a	*/
4250	n/a
4251	n/a	static int
4252	n/a	unicode_decode_call_errorhandler_wchar(
4253	n/a	const char errors, PyObject *errorHandler,
4254	n/a	const char encoding, const char reason,
4255	n/a	const char input, const char inend, Py_ssize_t *startinpos,
4256	n/a	Py_ssize_t endinpos, PyObject exceptionObject, const char *inptr,
4257	n/a	PyObject *output, Py_ssize_t outpos)
4258	n/a	{
4259	n/a	static const char *argparse = "Un;decoding error handler must return (str, int) tuple";
4260	n/a
4261	n/a	PyObject *restuple = NULL;
4262	n/a	PyObject *repunicode = NULL;
4263	n/a	Py_ssize_t outsize;
4264	n/a	Py_ssize_t insize;
4265	n/a	Py_ssize_t requiredsize;
4266	n/a	Py_ssize_t newpos;
4267	n/a	PyObject *inputobj = NULL;
4268	n/a	wchar_t *repwstr;
4269	n/a	Py_ssize_t repwlen;
4270	n/a
4271	n/a	assert (_PyUnicode_KIND(*output) == PyUnicode_WCHAR_KIND);
4272	n/a	outsize = _PyUnicode_WSTR_LENGTH(*output);
4273	n/a
4274	n/a	if (*errorHandler == NULL) {
4275	n/a	*errorHandler = PyCodec_LookupError(errors);
4276	n/a	if (*errorHandler == NULL)
4277	n/a	goto onError;
4278	n/a	}
4279	n/a
4280	n/a	make_decode_exception(exceptionObject,
4281	n/a	encoding,
4282	n/a	input, inend - *input,
4283	n/a	startinpos, endinpos,
4284	n/a	reason);
4285	n/a	if (*exceptionObject == NULL)
4286	n/a	goto onError;
4287	n/a
4288	n/a	restuple = PyObject_CallFunctionObjArgs(errorHandler, exceptionObject, NULL);
4289	n/a	if (restuple == NULL)
4290	n/a	goto onError;
4291	n/a	if (!PyTuple_Check(restuple)) {
4292	n/a	PyErr_SetString(PyExc_TypeError, &argparse[3]);
4293	n/a	goto onError;
4294	n/a	}
4295	n/a	if (!PyArg_ParseTuple(restuple, argparse, &repunicode, &newpos))
4296	n/a	goto onError;
4297	n/a
4298	n/a	/* Copy back the bytes variables, which might have been modified by the
4299	n/a	callback */
4300	n/a	inputobj = PyUnicodeDecodeError_GetObject(*exceptionObject);
4301	n/a	if (!inputobj)
4302	n/a	goto onError;
4303	n/a	*input = PyBytes_AS_STRING(inputobj);
4304	n/a	insize = PyBytes_GET_SIZE(inputobj);
4305	n/a	inend = input + insize;
4306	n/a	/* we can DECREF safely, as the exception has another reference,
4307	n/a	so the object won't go away. */
4308	n/a	Py_DECREF(inputobj);
4309	n/a
4310	n/a	if (newpos<0)
4311	n/a	newpos = insize+newpos;
4312	n/a	if (newpos<0 \|\| newpos>insize) {
4313	n/a	PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", newpos);
4314	n/a	goto onError;
4315	n/a	}
4316	n/a
4317	n/a	repwstr = PyUnicode_AsUnicodeAndSize(repunicode, &repwlen);
4318	n/a	if (repwstr == NULL)
4319	n/a	goto onError;
4320	n/a	/* need more space? (at least enough for what we
4321	n/a	have+the replacement+the rest of the string (starting
4322	n/a	at the new input position), so we won't have to check space
4323	n/a	when there are no errors in the rest of the string) */
4324	n/a	requiredsize = *outpos;
4325	n/a	if (requiredsize > PY_SSIZE_T_MAX - repwlen)
4326	n/a	goto overflow;
4327	n/a	requiredsize += repwlen;
4328	n/a	if (requiredsize > PY_SSIZE_T_MAX - (insize - newpos))
4329	n/a	goto overflow;
4330	n/a	requiredsize += insize - newpos;
4331	n/a	if (requiredsize > outsize) {
4332	n/a	if (outsize <= PY_SSIZE_T_MAX/2 && requiredsize < 2*outsize)
4333	n/a	requiredsize = 2*outsize;
4334	n/a	if (unicode_resize(output, requiredsize) < 0)
4335	n/a	goto onError;
4336	n/a	}
4337	n/a	wcsncpy(_PyUnicode_WSTR(output) + outpos, repwstr, repwlen);
4338	n/a	*outpos += repwlen;
4339	n/a	*endinpos = newpos;
4340	n/a	inptr = input + newpos;
4341	n/a
4342	n/a	/* we made it! */
4343	n/a	Py_DECREF(restuple);
4344	n/a	return 0;
4345	n/a
4346	n/a	overflow:
4347	n/a	PyErr_SetString(PyExc_OverflowError,
4348	n/a	"decoded result is too long for a Python string");
4349	n/a
4350	n/a	onError:
4351	n/a	Py_XDECREF(restuple);
4352	n/a	return -1;
4353	n/a	}
4354	n/a	#endif /* MS_WINDOWS */
4355	n/a
4356	n/a	static int
4357	n/a	unicode_decode_call_errorhandler_writer(
4358	n/a	const char errors, PyObject *errorHandler,
4359	n/a	const char encoding, const char reason,
4360	n/a	const char input, const char inend, Py_ssize_t *startinpos,
4361	n/a	Py_ssize_t endinpos, PyObject exceptionObject, const char *inptr,
4362	n/a	_PyUnicodeWriter writer / PyObject *output, Py_ssize_t outpos */)
4363	n/a	{
4364	n/a	static const char *argparse = "Un;decoding error handler must return (str, int) tuple";
4365	n/a
4366	n/a	PyObject *restuple = NULL;
4367	n/a	PyObject *repunicode = NULL;
4368	n/a	Py_ssize_t insize;
4369	n/a	Py_ssize_t newpos;
4370	n/a	Py_ssize_t replen;
4371	n/a	PyObject *inputobj = NULL;
4372	n/a
4373	n/a	if (*errorHandler == NULL) {
4374	n/a	*errorHandler = PyCodec_LookupError(errors);
4375	n/a	if (*errorHandler == NULL)
4376	n/a	goto onError;
4377	n/a	}
4378	n/a
4379	n/a	make_decode_exception(exceptionObject,
4380	n/a	encoding,
4381	n/a	input, inend - *input,
4382	n/a	startinpos, endinpos,
4383	n/a	reason);
4384	n/a	if (*exceptionObject == NULL)
4385	n/a	goto onError;
4386	n/a
4387	n/a	restuple = PyObject_CallFunctionObjArgs(errorHandler, exceptionObject, NULL);
4388	n/a	if (restuple == NULL)
4389	n/a	goto onError;
4390	n/a	if (!PyTuple_Check(restuple)) {
4391	n/a	PyErr_SetString(PyExc_TypeError, &argparse[3]);
4392	n/a	goto onError;
4393	n/a	}
4394	n/a	if (!PyArg_ParseTuple(restuple, argparse, &repunicode, &newpos))
4395	n/a	goto onError;
4396	n/a
4397	n/a	/* Copy back the bytes variables, which might have been modified by the
4398	n/a	callback */
4399	n/a	inputobj = PyUnicodeDecodeError_GetObject(*exceptionObject);
4400	n/a	if (!inputobj)
4401	n/a	goto onError;
4402	n/a	*input = PyBytes_AS_STRING(inputobj);
4403	n/a	insize = PyBytes_GET_SIZE(inputobj);
4404	n/a	inend = input + insize;
4405	n/a	/* we can DECREF safely, as the exception has another reference,
4406	n/a	so the object won't go away. */
4407	n/a	Py_DECREF(inputobj);
4408	n/a
4409	n/a	if (newpos<0)
4410	n/a	newpos = insize+newpos;
4411	n/a	if (newpos<0 \|\| newpos>insize) {
4412	n/a	PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", newpos);
4413	n/a	goto onError;
4414	n/a	}
4415	n/a
4416	n/a	replen = PyUnicode_GET_LENGTH(repunicode);
4417	n/a	if (replen > 1) {
4418	n/a	writer->min_length += replen - 1;
4419	n/a	writer->overallocate = 1;
4420	n/a	if (_PyUnicodeWriter_Prepare(writer, writer->min_length,
4421	n/a	PyUnicode_MAX_CHAR_VALUE(repunicode)) == -1)
4422	n/a	goto onError;
4423	n/a	}
4424	n/a	if (_PyUnicodeWriter_WriteStr(writer, repunicode) == -1)
4425	n/a	goto onError;
4426	n/a
4427	n/a	*endinpos = newpos;
4428	n/a	inptr = input + newpos;
4429	n/a
4430	n/a	/* we made it! */
4431	n/a	Py_DECREF(restuple);
4432	n/a	return 0;
4433	n/a
4434	n/a	onError:
4435	n/a	Py_XDECREF(restuple);
4436	n/a	return -1;
4437	n/a	}
4438	n/a
4439	n/a	/* --- UTF-7 Codec -------------------------------------------------------- */
4440	n/a
4441	n/a	/* See RFC2152 for details. We encode conservatively and decode liberally. */
4442	n/a
4443	n/a	/* Three simple macros defining base-64. */
4444	n/a
4445	n/a	/* Is c a base-64 character? */
4446	n/a
4447	n/a	#define IS_BASE64(c) \
4448	n/a	(((c) >= 'A' && (c) <= 'Z') \|\| \
4449	n/a	((c) >= 'a' && (c) <= 'z') \|\| \
4450	n/a	((c) >= '0' && (c) <= '9') \|\| \
4451	n/a	(c) == '+' \|\| (c) == '/')
4452	n/a
4453	n/a	/* given that c is a base-64 character, what is its base-64 value? */
4454	n/a
4455	n/a	#define FROM_BASE64(c) \
4456	n/a	(((c) >= 'A' && (c) <= 'Z') ? (c) - 'A' : \
4457	n/a	((c) >= 'a' && (c) <= 'z') ? (c) - 'a' + 26 : \
4458	n/a	((c) >= '0' && (c) <= '9') ? (c) - '0' + 52 : \
4459	n/a	(c) == '+' ? 62 : 63)
4460	n/a
4461	n/a	/* What is the base-64 character of the bottom 6 bits of n? */
4462	n/a
4463	n/a	#define TO_BASE64(n) \
4464	n/a	("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(n) & 0x3f])
4465	n/a
4466	n/a	/* DECODE_DIRECT: this byte encountered in a UTF-7 string should be
4467	n/a	* decoded as itself. We are permissive on decoding; the only ASCII
4468	n/a	* byte not decoding to itself is the + which begins a base64
4469	n/a	* string. */
4470	n/a
4471	n/a	#define DECODE_DIRECT(c) \
4472	n/a	((c) <= 127 && (c) != '+')
4473	n/a
4474	n/a	/* The UTF-7 encoder treats ASCII characters differently according to
4475	n/a	* whether they are Set D, Set O, Whitespace, or special (i.e. none of
4476	n/a	* the above). See RFC2152. This array identifies these different
4477	n/a	* sets:
4478	n/a	* 0 : "Set D"
4479	n/a	* alphanumeric and '(),-./:?
4480	n/a	* 1 : "Set O"
4481	n/a	* !"#$%&*;<=>@[]^_`{\|}
4482	n/a	* 2 : "whitespace"
4483	n/a	* ht nl cr sp
4484	n/a	* 3 : special (must be base64 encoded)
4485	n/a	* everything else (i.e. +\~ and non-printing codes 0-8 11-12 14-31 127)
4486	n/a	*/
4487	n/a
4488	n/a	static
4489	n/a	char utf7_category[128] = {
4490	n/a	/* nul soh stx etx eot enq ack bel bs ht nl vt np cr so si */
4491	n/a	3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 3, 3, 2, 3, 3,
4492	n/a	/* dle dc1 dc2 dc3 dc4 nak syn etb can em sub esc fs gs rs us */
4493	n/a	3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4494	n/a	/* sp ! " # $ % & ' ( ) * + , - . / */
4495	n/a	2, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 3, 0, 0, 0, 0,
4496	n/a	/* 0 1 2 3 4 5 6 7 8 9 : ; < = > ? */
4497	n/a	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0,
4498	n/a	/* @ A B C D E F G H I J K L M N O */
4499	n/a	1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4500	n/a	/* P Q R S T U V W X Y Z [ \ ] ^ _ */
4501	n/a	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1, 1, 1,
4502	n/a	/* ` a b c d e f g h i j k l m n o */
4503	n/a	1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4504	n/a	/* p q r s t u v w x y z { \| } ~ del */
4505	n/a	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 3, 3,
4506	n/a	};
4507	n/a
4508	n/a	/* ENCODE_DIRECT: this character should be encoded as itself. The
4509	n/a	* answer depends on whether we are encoding set O as itself, and also
4510	n/a	* on whether we are encoding whitespace as itself. RFC2152 makes it
4511	n/a	* clear that the answers to these questions vary between
4512	n/a	* applications, so this code needs to be flexible. */
4513	n/a
4514	n/a	#define ENCODE_DIRECT(c, directO, directWS) \
4515	n/a	((c) < 128 && (c) > 0 && \
4516	n/a	((utf7_category[(c)] == 0) \|\| \
4517	n/a	(directWS && (utf7_category[(c)] == 2)) \|\| \
4518	n/a	(directO && (utf7_category[(c)] == 1))))
4519	n/a
4520	n/a	PyObject *
4521	n/a	PyUnicode_DecodeUTF7(const char *s,
4522	n/a	Py_ssize_t size,
4523	n/a	const char *errors)
4524	n/a	{
4525	n/a	return PyUnicode_DecodeUTF7Stateful(s, size, errors, NULL);
4526	n/a	}
4527	n/a
4528	n/a	/* The decoder. The only state we preserve is our read position,
4529	n/a	* i.e. how many characters we have consumed. So if we end in the
4530	n/a	* middle of a shift sequence we have to back off the read position
4531	n/a	* and the output to the beginning of the sequence, otherwise we lose
4532	n/a	* all the shift state (seen bits, number of bits seen, high
4533	n/a	* surrogate). */
4534	n/a
4535	n/a	PyObject *
4536	n/a	PyUnicode_DecodeUTF7Stateful(const char *s,
4537	n/a	Py_ssize_t size,
4538	n/a	const char *errors,
4539	n/a	Py_ssize_t *consumed)
4540	n/a	{
4541	n/a	const char *starts = s;
4542	n/a	Py_ssize_t startinpos;
4543	n/a	Py_ssize_t endinpos;
4544	n/a	const char *e;
4545	n/a	_PyUnicodeWriter writer;
4546	n/a	const char *errmsg = "";
4547	n/a	int inShift = 0;
4548	n/a	Py_ssize_t shiftOutStart;
4549	n/a	unsigned int base64bits = 0;
4550	n/a	unsigned long base64buffer = 0;
4551	n/a	Py_UCS4 surrogate = 0;
4552	n/a	PyObject *errorHandler = NULL;
4553	n/a	PyObject *exc = NULL;
4554	n/a
4555	n/a	if (size == 0) {
4556	n/a	if (consumed)
4557	n/a	*consumed = 0;
4558	n/a	_Py_RETURN_UNICODE_EMPTY();
4559	n/a	}
4560	n/a
4561	n/a	/* Start off assuming it's all ASCII. Widen later as necessary. */
4562	n/a	_PyUnicodeWriter_Init(&writer);
4563	n/a	writer.min_length = size;
4564	n/a
4565	n/a	shiftOutStart = 0;
4566	n/a	e = s + size;
4567	n/a
4568	n/a	while (s < e) {
4569	n/a	Py_UCS4 ch;
4570	n/a	restart:
4571	n/a	ch = (unsigned char) *s;
4572	n/a
4573	n/a	if (inShift) { /* in a base-64 section */
4574	n/a	if (IS_BASE64(ch)) { /* consume a base-64 character */
4575	n/a	base64buffer = (base64buffer << 6) \| FROM_BASE64(ch);
4576	n/a	base64bits += 6;
4577	n/a	s++;
4578	n/a	if (base64bits >= 16) {
4579	n/a	/* we have enough bits for a UTF-16 value */
4580	n/a	Py_UCS4 outCh = (Py_UCS4)(base64buffer >> (base64bits-16));
4581	n/a	base64bits -= 16;
4582	n/a	base64buffer &= (1 << base64bits) - 1; /* clear high bits */
4583	n/a	assert(outCh <= 0xffff);
4584	n/a	if (surrogate) {
4585	n/a	/* expecting a second surrogate */
4586	n/a	if (Py_UNICODE_IS_LOW_SURROGATE(outCh)) {
4587	n/a	Py_UCS4 ch2 = Py_UNICODE_JOIN_SURROGATES(surrogate, outCh);
4588	n/a	if (_PyUnicodeWriter_WriteCharInline(&writer, ch2) < 0)
4589	n/a	goto onError;
4590	n/a	surrogate = 0;
4591	n/a	continue;
4592	n/a	}
4593	n/a	else {
4594	n/a	if (_PyUnicodeWriter_WriteCharInline(&writer, surrogate) < 0)
4595	n/a	goto onError;
4596	n/a	surrogate = 0;
4597	n/a	}
4598	n/a	}
4599	n/a	if (Py_UNICODE_IS_HIGH_SURROGATE(outCh)) {
4600	n/a	/* first surrogate */
4601	n/a	surrogate = outCh;
4602	n/a	}
4603	n/a	else {
4604	n/a	if (_PyUnicodeWriter_WriteCharInline(&writer, outCh) < 0)
4605	n/a	goto onError;
4606	n/a	}
4607	n/a	}
4608	n/a	}
4609	n/a	else { /* now leaving a base-64 section */
4610	n/a	inShift = 0;
4611	n/a	if (base64bits > 0) { /* left-over bits */
4612	n/a	if (base64bits >= 6) {
4613	n/a	/* We've seen at least one base-64 character */
4614	n/a	s++;
4615	n/a	errmsg = "partial character in shift sequence";
4616	n/a	goto utf7Error;
4617	n/a	}
4618	n/a	else {
4619	n/a	/* Some bits remain; they should be zero */
4620	n/a	if (base64buffer != 0) {
4621	n/a	s++;
4622	n/a	errmsg = "non-zero padding bits in shift sequence";
4623	n/a	goto utf7Error;
4624	n/a	}
4625	n/a	}
4626	n/a	}
4627	n/a	if (surrogate && DECODE_DIRECT(ch)) {
4628	n/a	if (_PyUnicodeWriter_WriteCharInline(&writer, surrogate) < 0)
4629	n/a	goto onError;
4630	n/a	}
4631	n/a	surrogate = 0;
4632	n/a	if (ch == '-') {
4633	n/a	/* '-' is absorbed; other terminating
4634	n/a	characters are preserved */
4635	n/a	s++;
4636	n/a	}
4637	n/a	}
4638	n/a	}
4639	n/a	else if ( ch == '+' ) {
4640	n/a	startinpos = s-starts;
4641	n/a	s++; /* consume '+' */
4642	n/a	if (s < e && s == '-') { / '+-' encodes '+' */
4643	n/a	s++;
4644	n/a	if (_PyUnicodeWriter_WriteCharInline(&writer, '+') < 0)
4645	n/a	goto onError;
4646	n/a	}
4647	n/a	else { /* begin base64-encoded section */
4648	n/a	inShift = 1;
4649	n/a	surrogate = 0;
4650	n/a	shiftOutStart = writer.pos;
4651	n/a	base64bits = 0;
4652	n/a	base64buffer = 0;
4653	n/a	}
4654	n/a	}
4655	n/a	else if (DECODE_DIRECT(ch)) { /* character decodes as itself */
4656	n/a	s++;
4657	n/a	if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0)
4658	n/a	goto onError;
4659	n/a	}
4660	n/a	else {
4661	n/a	startinpos = s-starts;
4662	n/a	s++;
4663	n/a	errmsg = "unexpected special character";
4664	n/a	goto utf7Error;
4665	n/a	}
4666	n/a	continue;
4667	n/a	utf7Error:
4668	n/a	endinpos = s-starts;
4669	n/a	if (unicode_decode_call_errorhandler_writer(
4670	n/a	errors, &errorHandler,
4671	n/a	"utf7", errmsg,
4672	n/a	&starts, &e, &startinpos, &endinpos, &exc, &s,
4673	n/a	&writer))
4674	n/a	goto onError;
4675	n/a	}
4676	n/a
4677	n/a	/* end of string */
4678	n/a
4679	n/a	if (inShift && !consumed) { /* in shift sequence, no more to follow */
4680	n/a	/* if we're in an inconsistent state, that's an error */
4681	n/a	inShift = 0;
4682	n/a	if (surrogate \|\|
4683	n/a	(base64bits >= 6) \|\|
4684	n/a	(base64bits > 0 && base64buffer != 0)) {
4685	n/a	endinpos = size;
4686	n/a	if (unicode_decode_call_errorhandler_writer(
4687	n/a	errors, &errorHandler,
4688	n/a	"utf7", "unterminated shift sequence",
4689	n/a	&starts, &e, &startinpos, &endinpos, &exc, &s,
4690	n/a	&writer))
4691	n/a	goto onError;
4692	n/a	if (s < e)
4693	n/a	goto restart;
4694	n/a	}
4695	n/a	}
4696	n/a
4697	n/a	/* return state */
4698	n/a	if (consumed) {
4699	n/a	if (inShift) {
4700	n/a	*consumed = startinpos;
4701	n/a	if (writer.pos != shiftOutStart && writer.maxchar > 127) {
4702	n/a	PyObject *result = PyUnicode_FromKindAndData(
4703	n/a	writer.kind, writer.data, shiftOutStart);
4704	n/a	Py_XDECREF(errorHandler);
4705	n/a	Py_XDECREF(exc);
4706	n/a	_PyUnicodeWriter_Dealloc(&writer);
4707	n/a	return result;
4708	n/a	}
4709	n/a	writer.pos = shiftOutStart; /* back off output */
4710	n/a	}
4711	n/a	else {
4712	n/a	*consumed = s-starts;
4713	n/a	}
4714	n/a	}
4715	n/a
4716	n/a	Py_XDECREF(errorHandler);
4717	n/a	Py_XDECREF(exc);
4718	n/a	return _PyUnicodeWriter_Finish(&writer);
4719	n/a
4720	n/a	onError:
4721	n/a	Py_XDECREF(errorHandler);
4722	n/a	Py_XDECREF(exc);
4723	n/a	_PyUnicodeWriter_Dealloc(&writer);
4724	n/a	return NULL;
4725	n/a	}
4726	n/a
4727	n/a
4728	n/a	PyObject *
4729	n/a	_PyUnicode_EncodeUTF7(PyObject *str,
4730	n/a	int base64SetO,
4731	n/a	int base64WhiteSpace,
4732	n/a	const char *errors)
4733	n/a	{
4734	n/a	int kind;
4735	n/a	void *data;
4736	n/a	Py_ssize_t len;
4737	n/a	PyObject *v;
4738	n/a	int inShift = 0;
4739	n/a	Py_ssize_t i;
4740	n/a	unsigned int base64bits = 0;
4741	n/a	unsigned long base64buffer = 0;
4742	n/a	char * out;
4743	n/a	char * start;
4744	n/a
4745	n/a	if (PyUnicode_READY(str) == -1)
4746	n/a	return NULL;
4747	n/a	kind = PyUnicode_KIND(str);
4748	n/a	data = PyUnicode_DATA(str);
4749	n/a	len = PyUnicode_GET_LENGTH(str);
4750	n/a
4751	n/a	if (len == 0)
4752	n/a	return PyBytes_FromStringAndSize(NULL, 0);
4753	n/a
4754	n/a	/* It might be possible to tighten this worst case */
4755	n/a	if (len > PY_SSIZE_T_MAX / 8)
4756	n/a	return PyErr_NoMemory();
4757	n/a	v = PyBytes_FromStringAndSize(NULL, len * 8);
4758	n/a	if (v == NULL)
4759	n/a	return NULL;
4760	n/a
4761	n/a	start = out = PyBytes_AS_STRING(v);
4762	n/a	for (i = 0; i < len; ++i) {
4763	n/a	Py_UCS4 ch = PyUnicode_READ(kind, data, i);
4764	n/a
4765	n/a	if (inShift) {
4766	n/a	if (ENCODE_DIRECT(ch, !base64SetO, !base64WhiteSpace)) {
4767	n/a	/* shifting out */
4768	n/a	if (base64bits) { /* output remaining bits */
4769	n/a	*out++ = TO_BASE64(base64buffer << (6-base64bits));
4770	n/a	base64buffer = 0;
4771	n/a	base64bits = 0;
4772	n/a	}
4773	n/a	inShift = 0;
4774	n/a	/* Characters not in the BASE64 set implicitly unshift the sequence
4775	n/a	so no '-' is required, except if the character is itself a '-' */
4776	n/a	if (IS_BASE64(ch) \|\| ch == '-') {
4777	n/a	*out++ = '-';
4778	n/a	}
4779	n/a	*out++ = (char) ch;
4780	n/a	}
4781	n/a	else {
4782	n/a	goto encode_char;
4783	n/a	}
4784	n/a	}
4785	n/a	else { /* not in a shift sequence */
4786	n/a	if (ch == '+') {
4787	n/a	*out++ = '+';
4788	n/a	*out++ = '-';
4789	n/a	}
4790	n/a	else if (ENCODE_DIRECT(ch, !base64SetO, !base64WhiteSpace)) {
4791	n/a	*out++ = (char) ch;
4792	n/a	}
4793	n/a	else {
4794	n/a	*out++ = '+';
4795	n/a	inShift = 1;
4796	n/a	goto encode_char;
4797	n/a	}
4798	n/a	}
4799	n/a	continue;
4800	n/a	encode_char:
4801	n/a	if (ch >= 0x10000) {
4802	n/a	assert(ch <= MAX_UNICODE);
4803	n/a
4804	n/a	/* code first surrogate */
4805	n/a	base64bits += 16;
4806	n/a	base64buffer = (base64buffer << 16) \| Py_UNICODE_HIGH_SURROGATE(ch);
4807	n/a	while (base64bits >= 6) {
4808	n/a	*out++ = TO_BASE64(base64buffer >> (base64bits-6));
4809	n/a	base64bits -= 6;
4810	n/a	}
4811	n/a	/* prepare second surrogate */
4812	n/a	ch = Py_UNICODE_LOW_SURROGATE(ch);
4813	n/a	}
4814	n/a	base64bits += 16;
4815	n/a	base64buffer = (base64buffer << 16) \| ch;
4816	n/a	while (base64bits >= 6) {
4817	n/a	*out++ = TO_BASE64(base64buffer >> (base64bits-6));
4818	n/a	base64bits -= 6;
4819	n/a	}
4820	n/a	}
4821	n/a	if (base64bits)
4822	n/a	*out++= TO_BASE64(base64buffer << (6-base64bits) );
4823	n/a	if (inShift)
4824	n/a	*out++ = '-';
4825	n/a	if (_PyBytes_Resize(&v, out - start) < 0)
4826	n/a	return NULL;
4827	n/a	return v;
4828	n/a	}
4829	n/a	PyObject *
4830	n/a	PyUnicode_EncodeUTF7(const Py_UNICODE *s,
4831	n/a	Py_ssize_t size,
4832	n/a	int base64SetO,
4833	n/a	int base64WhiteSpace,
4834	n/a	const char *errors)
4835	n/a	{
4836	n/a	PyObject *result;
4837	n/a	PyObject *tmp = PyUnicode_FromWideChar(s, size);
4838	n/a	if (tmp == NULL)
4839	n/a	return NULL;
4840	n/a	result = _PyUnicode_EncodeUTF7(tmp, base64SetO,
4841	n/a	base64WhiteSpace, errors);
4842	n/a	Py_DECREF(tmp);
4843	n/a	return result;
4844	n/a	}
4845	n/a
4846	n/a	#undef IS_BASE64
4847	n/a	#undef FROM_BASE64
4848	n/a	#undef TO_BASE64
4849	n/a	#undef DECODE_DIRECT
4850	n/a	#undef ENCODE_DIRECT
4851	n/a
4852	n/a	/* --- UTF-8 Codec -------------------------------------------------------- */
4853	n/a
4854	n/a	PyObject *
4855	n/a	PyUnicode_DecodeUTF8(const char *s,
4856	n/a	Py_ssize_t size,
4857	n/a	const char *errors)
4858	n/a	{
4859	n/a	return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL);
4860	n/a	}
4861	n/a
4862	n/a	#include "stringlib/asciilib.h"
4863	n/a	#include "stringlib/codecs.h"
4864	n/a	#include "stringlib/undef.h"
4865	n/a
4866	n/a	#include "stringlib/ucs1lib.h"
4867	n/a	#include "stringlib/codecs.h"
4868	n/a	#include "stringlib/undef.h"
4869	n/a
4870	n/a	#include "stringlib/ucs2lib.h"
4871	n/a	#include "stringlib/codecs.h"
4872	n/a	#include "stringlib/undef.h"
4873	n/a
4874	n/a	#include "stringlib/ucs4lib.h"
4875	n/a	#include "stringlib/codecs.h"
4876	n/a	#include "stringlib/undef.h"
4877	n/a
4878	n/a	/* Mask to quickly check whether a C 'long' contains a
4879	n/a	non-ASCII, UTF8-encoded char. */
4880	n/a	#if (SIZEOF_LONG == 8)
4881	n/a	# define ASCII_CHAR_MASK 0x8080808080808080UL
4882	n/a	#elif (SIZEOF_LONG == 4)
4883	n/a	# define ASCII_CHAR_MASK 0x80808080UL
4884	n/a	#else
4885	n/a	# error C 'long' size should be either 4 or 8!
4886	n/a	#endif
4887	n/a
4888	n/a	static Py_ssize_t
4889	n/a	ascii_decode(const char start, const char end, Py_UCS1 *dest)
4890	n/a	{
4891	n/a	const char *p = start;
4892	n/a	const char aligned_end = (const char ) _Py_ALIGN_DOWN(end, SIZEOF_LONG);
4893	n/a
4894	n/a	/*
4895	n/a	* Issue #17237: m68k is a bit different from most architectures in
4896	n/a	* that objects do not use "natural alignment" - for example, int and
4897	n/a	* long are only aligned at 2-byte boundaries. Therefore the assert()
4898	n/a	* won't work; also, tests have shown that skipping the "optimised
4899	n/a	* version" will even speed up m68k.
4900	n/a	*/
4901	n/a	#if !defined(__m68k__)
4902	n/a	#if SIZEOF_LONG <= SIZEOF_VOID_P
4903	n/a	assert(_Py_IS_ALIGNED(dest, SIZEOF_LONG));
4904	n/a	if (_Py_IS_ALIGNED(p, SIZEOF_LONG)) {
4905	n/a	/* Fast path, see in STRINGLIB(utf8_decode) for
4906	n/a	an explanation. */
4907	n/a	/* Help allocation */
4908	n/a	const char *_p = p;
4909	n/a	Py_UCS1 * q = dest;
4910	n/a	while (_p < aligned_end) {
4911	n/a	unsigned long value = (const unsigned long ) _p;
4912	n/a	if (value & ASCII_CHAR_MASK)
4913	n/a	break;
4914	n/a	((unsigned long )q) = value;
4915	n/a	_p += SIZEOF_LONG;
4916	n/a	q += SIZEOF_LONG;
4917	n/a	}
4918	n/a	p = _p;
4919	n/a	while (p < end) {
4920	n/a	if ((unsigned char)*p & 0x80)
4921	n/a	break;
4922	n/a	q++ = p++;
4923	n/a	}
4924	n/a	return p - start;
4925	n/a	}
4926	n/a	#endif
4927	n/a	#endif
4928	n/a	while (p < end) {
4929	n/a	/* Fast path, see in STRINGLIB(utf8_decode) in stringlib/codecs.h
4930	n/a	for an explanation. */
4931	n/a	if (_Py_IS_ALIGNED(p, SIZEOF_LONG)) {
4932	n/a	/* Help allocation */
4933	n/a	const char *_p = p;
4934	n/a	while (_p < aligned_end) {
4935	n/a	unsigned long value = (unsigned long ) _p;
4936	n/a	if (value & ASCII_CHAR_MASK)
4937	n/a	break;
4938	n/a	_p += SIZEOF_LONG;
4939	n/a	}
4940	n/a	p = _p;
4941	n/a	if (_p == end)
4942	n/a	break;
4943	n/a	}
4944	n/a	if ((unsigned char)*p & 0x80)
4945	n/a	break;
4946	n/a	++p;
4947	n/a	}
4948	n/a	memcpy(dest, start, p - start);
4949	n/a	return p - start;
4950	n/a	}
4951	n/a
4952	n/a	PyObject *
4953	n/a	PyUnicode_DecodeUTF8Stateful(const char *s,
4954	n/a	Py_ssize_t size,
4955	n/a	const char *errors,
4956	n/a	Py_ssize_t *consumed)
4957	n/a	{
4958	n/a	_PyUnicodeWriter writer;
4959	n/a	const char *starts = s;
4960	n/a	const char *end = s + size;
4961	n/a
4962	n/a	Py_ssize_t startinpos;
4963	n/a	Py_ssize_t endinpos;
4964	n/a	const char *errmsg = "";
4965	n/a	PyObject *error_handler_obj = NULL;
4966	n/a	PyObject *exc = NULL;
4967	n/a	_Py_error_handler error_handler = _Py_ERROR_UNKNOWN;
4968	n/a
4969	n/a	if (size == 0) {
4970	n/a	if (consumed)
4971	n/a	*consumed = 0;
4972	n/a	_Py_RETURN_UNICODE_EMPTY();
4973	n/a	}
4974	n/a
4975	n/a	/* ASCII is equivalent to the first 128 ordinals in Unicode. */
4976	n/a	if (size == 1 && (unsigned char)s[0] < 128) {
4977	n/a	if (consumed)
4978	n/a	*consumed = 1;
4979	n/a	return get_latin1_char((unsigned char)s[0]);
4980	n/a	}
4981	n/a
4982	n/a	_PyUnicodeWriter_Init(&writer);
4983	n/a	writer.min_length = size;
4984	n/a	if (_PyUnicodeWriter_Prepare(&writer, writer.min_length, 127) == -1)
4985	n/a	goto onError;
4986	n/a
4987	n/a	writer.pos = ascii_decode(s, end, writer.data);
4988	n/a	s += writer.pos;
4989	n/a	while (s < end) {
4990	n/a	Py_UCS4 ch;
4991	n/a	int kind = writer.kind;
4992	n/a
4993	n/a	if (kind == PyUnicode_1BYTE_KIND) {
4994	n/a	if (PyUnicode_IS_ASCII(writer.buffer))
4995	n/a	ch = asciilib_utf8_decode(&s, end, writer.data, &writer.pos);
4996	n/a	else
4997	n/a	ch = ucs1lib_utf8_decode(&s, end, writer.data, &writer.pos);
4998	n/a	} else if (kind == PyUnicode_2BYTE_KIND) {
4999	n/a	ch = ucs2lib_utf8_decode(&s, end, writer.data, &writer.pos);
5000	n/a	} else {
5001	n/a	assert(kind == PyUnicode_4BYTE_KIND);
5002	n/a	ch = ucs4lib_utf8_decode(&s, end, writer.data, &writer.pos);
5003	n/a	}
5004	n/a
5005	n/a	switch (ch) {
5006	n/a	case 0:
5007	n/a	if (s == end \|\| consumed)
5008	n/a	goto End;
5009	n/a	errmsg = "unexpected end of data";
5010	n/a	startinpos = s - starts;
5011	n/a	endinpos = end - starts;
5012	n/a	break;
5013	n/a	case 1:
5014	n/a	errmsg = "invalid start byte";
5015	n/a	startinpos = s - starts;
5016	n/a	endinpos = startinpos + 1;
5017	n/a	break;
5018	n/a	case 2:
5019	n/a	case 3:
5020	n/a	case 4:
5021	n/a	errmsg = "invalid continuation byte";
5022	n/a	startinpos = s - starts;
5023	n/a	endinpos = startinpos + ch - 1;
5024	n/a	break;
5025	n/a	default:
5026	n/a	if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0)
5027	n/a	goto onError;
5028	n/a	continue;
5029	n/a	}
5030	n/a
5031	n/a	if (error_handler == _Py_ERROR_UNKNOWN)
5032	n/a	error_handler = get_error_handler(errors);
5033	n/a
5034	n/a	switch (error_handler) {
5035	n/a	case _Py_ERROR_IGNORE:
5036	n/a	s += (endinpos - startinpos);
5037	n/a	break;
5038	n/a
5039	n/a	case _Py_ERROR_REPLACE:
5040	n/a	if (_PyUnicodeWriter_WriteCharInline(&writer, 0xfffd) < 0)
5041	n/a	goto onError;
5042	n/a	s += (endinpos - startinpos);
5043	n/a	break;
5044	n/a
5045	n/a	case _Py_ERROR_SURROGATEESCAPE:
5046	n/a	{
5047	n/a	Py_ssize_t i;
5048	n/a
5049	n/a	if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0)
5050	n/a	goto onError;
5051	n/a	for (i=startinpos; i<endinpos; i++) {
5052	n/a	ch = (Py_UCS4)(unsigned char)(starts[i]);
5053	n/a	PyUnicode_WRITE(writer.kind, writer.data, writer.pos,
5054	n/a	ch + 0xdc00);
5055	n/a	writer.pos++;
5056	n/a	}
5057	n/a	s += (endinpos - startinpos);
5058	n/a	break;
5059	n/a	}
5060	n/a
5061	n/a	default:
5062	n/a	if (unicode_decode_call_errorhandler_writer(
5063	n/a	errors, &error_handler_obj,
5064	n/a	"utf-8", errmsg,
5065	n/a	&starts, &end, &startinpos, &endinpos, &exc, &s,
5066	n/a	&writer))
5067	n/a	goto onError;
5068	n/a	}
5069	n/a	}
5070	n/a
5071	n/a	End:
5072	n/a	if (consumed)
5073	n/a	*consumed = s - starts;
5074	n/a
5075	n/a	Py_XDECREF(error_handler_obj);
5076	n/a	Py_XDECREF(exc);
5077	n/a	return _PyUnicodeWriter_Finish(&writer);
5078	n/a
5079	n/a	onError:
5080	n/a	Py_XDECREF(error_handler_obj);
5081	n/a	Py_XDECREF(exc);
5082	n/a	_PyUnicodeWriter_Dealloc(&writer);
5083	n/a	return NULL;
5084	n/a	}
5085	n/a
5086	n/a	#if defined(__APPLE__) \|\| defined(__ANDROID__)
5087	n/a
5088	n/a	/* Simplified UTF-8 decoder using surrogateescape error handler,
5089	n/a	used to decode the command line arguments on Mac OS X and Android.
5090	n/a
5091	n/a	Return a pointer to a newly allocated wide character string (use
5092	n/a	PyMem_RawFree() to free the memory), or NULL on memory allocation error. */
5093	n/a
5094	n/a	wchar_t*
5095	n/a	_Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size)
5096	n/a	{
5097	n/a	const char *e;
5098	n/a	wchar_t *unicode;
5099	n/a	Py_ssize_t outpos;
5100	n/a
5101	n/a	/* Note: size will always be longer than the resulting Unicode
5102	n/a	character count */
5103	n/a	if (PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(wchar_t) < (size + 1))
5104	n/a	return NULL;
5105	n/a	unicode = PyMem_RawMalloc((size + 1) * sizeof(wchar_t));
5106	n/a	if (!unicode)
5107	n/a	return NULL;
5108	n/a
5109	n/a	/* Unpack UTF-8 encoded data */
5110	n/a	e = s + size;
5111	n/a	outpos = 0;
5112	n/a	while (s < e) {
5113	n/a	Py_UCS4 ch;
5114	n/a	#if SIZEOF_WCHAR_T == 4
5115	n/a	ch = ucs4lib_utf8_decode(&s, e, (Py_UCS4 *)unicode, &outpos);
5116	n/a	#else
5117	n/a	ch = ucs2lib_utf8_decode(&s, e, (Py_UCS2 *)unicode, &outpos);
5118	n/a	#endif
5119	n/a	if (ch > 0xFF) {
5120	n/a	#if SIZEOF_WCHAR_T == 4
5121	n/a	assert(0);
5122	n/a	#else
5123	n/a	assert(ch > 0xFFFF && ch <= MAX_UNICODE);
5124	n/a	/* compute and append the two surrogates: */
5125	n/a	unicode[outpos++] = (wchar_t)Py_UNICODE_HIGH_SURROGATE(ch);
5126	n/a	unicode[outpos++] = (wchar_t)Py_UNICODE_LOW_SURROGATE(ch);
5127	n/a	#endif
5128	n/a	}
5129	n/a	else {
5130	n/a	if (!ch && s == e)
5131	n/a	break;
5132	n/a	/* surrogateescape */
5133	n/a	unicode[outpos++] = 0xDC00 + (unsigned char)*s++;
5134	n/a	}
5135	n/a	}
5136	n/a	unicode[outpos] = L'\0';
5137	n/a	return unicode;
5138	n/a	}
5139	n/a
5140	n/a	#endif /* __APPLE__ or __ANDROID__ */
5141	n/a
5142	n/a	/* Primary internal function which creates utf8 encoded bytes objects.
5143	n/a
5144	n/a	Allocation strategy: if the string is short, convert into a stack buffer
5145	n/a	and allocate exactly as much space needed at the end. Else allocate the
5146	n/a	maximum possible needed (4 result bytes per Unicode character), and return
5147	n/a	the excess memory at the end.
5148	n/a	*/
5149	n/a	PyObject *
5150	n/a	_PyUnicode_AsUTF8String(PyObject unicode, const char errors)
5151	n/a	{
5152	n/a	enum PyUnicode_Kind kind;
5153	n/a	void *data;
5154	n/a	Py_ssize_t size;
5155	n/a
5156	n/a	if (!PyUnicode_Check(unicode)) {
5157	n/a	PyErr_BadArgument();
5158	n/a	return NULL;
5159	n/a	}
5160	n/a
5161	n/a	if (PyUnicode_READY(unicode) == -1)
5162	n/a	return NULL;
5163	n/a
5164	n/a	if (PyUnicode_UTF8(unicode))
5165	n/a	return PyBytes_FromStringAndSize(PyUnicode_UTF8(unicode),
5166	n/a	PyUnicode_UTF8_LENGTH(unicode));
5167	n/a
5168	n/a	kind = PyUnicode_KIND(unicode);
5169	n/a	data = PyUnicode_DATA(unicode);
5170	n/a	size = PyUnicode_GET_LENGTH(unicode);
5171	n/a
5172	n/a	switch (kind) {
5173	n/a	default:
5174	n/a	assert(0);
5175	n/a	case PyUnicode_1BYTE_KIND:
5176	n/a	/* the string cannot be ASCII, or PyUnicode_UTF8() would be set */
5177	n/a	assert(!PyUnicode_IS_ASCII(unicode));
5178	n/a	return ucs1lib_utf8_encoder(unicode, data, size, errors);
5179	n/a	case PyUnicode_2BYTE_KIND:
5180	n/a	return ucs2lib_utf8_encoder(unicode, data, size, errors);
5181	n/a	case PyUnicode_4BYTE_KIND:
5182	n/a	return ucs4lib_utf8_encoder(unicode, data, size, errors);
5183	n/a	}
5184	n/a	}
5185	n/a
5186	n/a	PyObject *
5187	n/a	PyUnicode_EncodeUTF8(const Py_UNICODE *s,
5188	n/a	Py_ssize_t size,
5189	n/a	const char *errors)
5190	n/a	{
5191	n/a	PyObject v, unicode;
5192	n/a
5193	n/a	unicode = PyUnicode_FromWideChar(s, size);
5194	n/a	if (unicode == NULL)
5195	n/a	return NULL;
5196	n/a	v = _PyUnicode_AsUTF8String(unicode, errors);
5197	n/a	Py_DECREF(unicode);
5198	n/a	return v;
5199	n/a	}
5200	n/a
5201	n/a	PyObject *
5202	n/a	PyUnicode_AsUTF8String(PyObject *unicode)
5203	n/a	{
5204	n/a	return _PyUnicode_AsUTF8String(unicode, NULL);
5205	n/a	}
5206	n/a
5207	n/a	/* --- UTF-32 Codec ------------------------------------------------------- */
5208	n/a
5209	n/a	PyObject *
5210	n/a	PyUnicode_DecodeUTF32(const char *s,
5211	n/a	Py_ssize_t size,
5212	n/a	const char *errors,
5213	n/a	int *byteorder)
5214	n/a	{
5215	n/a	return PyUnicode_DecodeUTF32Stateful(s, size, errors, byteorder, NULL);
5216	n/a	}
5217	n/a
5218	n/a	PyObject *
5219	n/a	PyUnicode_DecodeUTF32Stateful(const char *s,
5220	n/a	Py_ssize_t size,
5221	n/a	const char *errors,
5222	n/a	int *byteorder,
5223	n/a	Py_ssize_t *consumed)
5224	n/a	{
5225	n/a	const char *starts = s;
5226	n/a	Py_ssize_t startinpos;
5227	n/a	Py_ssize_t endinpos;
5228	n/a	_PyUnicodeWriter writer;
5229	n/a	const unsigned char q, e;
5230	n/a	int le, bo = 0; /* assume native ordering by default */
5231	n/a	const char *encoding;
5232	n/a	const char *errmsg = "";
5233	n/a	PyObject *errorHandler = NULL;
5234	n/a	PyObject *exc = NULL;
5235	n/a
5236	n/a	q = (unsigned char *)s;
5237	n/a	e = q + size;
5238	n/a
5239	n/a	if (byteorder)
5240	n/a	bo = *byteorder;
5241	n/a
5242	n/a	/* Check for BOM marks (U+FEFF) in the input and adjust current
5243	n/a	byte order setting accordingly. In native mode, the leading BOM
5244	n/a	mark is skipped, in all other modes, it is copied to the output
5245	n/a	stream as-is (giving a ZWNBSP character). */
5246	n/a	if (bo == 0 && size >= 4) {
5247	n/a	Py_UCS4 bom = ((unsigned int)q[3] << 24) \| (q[2] << 16) \| (q[1] << 8) \| q[0];
5248	n/a	if (bom == 0x0000FEFF) {
5249	n/a	bo = -1;
5250	n/a	q += 4;
5251	n/a	}
5252	n/a	else if (bom == 0xFFFE0000) {
5253	n/a	bo = 1;
5254	n/a	q += 4;
5255	n/a	}
5256	n/a	if (byteorder)
5257	n/a	*byteorder = bo;
5258	n/a	}
5259	n/a
5260	n/a	if (q == e) {
5261	n/a	if (consumed)
5262	n/a	*consumed = size;
5263	n/a	_Py_RETURN_UNICODE_EMPTY();
5264	n/a	}
5265	n/a
5266	n/a	#ifdef WORDS_BIGENDIAN
5267	n/a	le = bo < 0;
5268	n/a	#else
5269	n/a	le = bo <= 0;
5270	n/a	#endif
5271	n/a	encoding = le ? "utf-32-le" : "utf-32-be";
5272	n/a
5273	n/a	_PyUnicodeWriter_Init(&writer);
5274	n/a	writer.min_length = (e - q + 3) / 4;
5275	n/a	if (_PyUnicodeWriter_Prepare(&writer, writer.min_length, 127) == -1)
5276	n/a	goto onError;
5277	n/a
5278	n/a	while (1) {
5279	n/a	Py_UCS4 ch = 0;
5280	n/a	Py_UCS4 maxch = PyUnicode_MAX_CHAR_VALUE(writer.buffer);
5281	n/a
5282	n/a	if (e - q >= 4) {
5283	n/a	enum PyUnicode_Kind kind = writer.kind;
5284	n/a	void *data = writer.data;
5285	n/a	const unsigned char *last = e - 4;
5286	n/a	Py_ssize_t pos = writer.pos;
5287	n/a	if (le) {
5288	n/a	do {
5289	n/a	ch = ((unsigned int)q[3] << 24) \| (q[2] << 16) \| (q[1] << 8) \| q[0];
5290	n/a	if (ch > maxch)
5291	n/a	break;
5292	n/a	if (kind != PyUnicode_1BYTE_KIND &&
5293	n/a	Py_UNICODE_IS_SURROGATE(ch))
5294	n/a	break;
5295	n/a	PyUnicode_WRITE(kind, data, pos++, ch);
5296	n/a	q += 4;
5297	n/a	} while (q <= last);
5298	n/a	}
5299	n/a	else {
5300	n/a	do {
5301	n/a	ch = ((unsigned int)q[0] << 24) \| (q[1] << 16) \| (q[2] << 8) \| q[3];
5302	n/a	if (ch > maxch)
5303	n/a	break;
5304	n/a	if (kind != PyUnicode_1BYTE_KIND &&
5305	n/a	Py_UNICODE_IS_SURROGATE(ch))
5306	n/a	break;
5307	n/a	PyUnicode_WRITE(kind, data, pos++, ch);
5308	n/a	q += 4;
5309	n/a	} while (q <= last);
5310	n/a	}
5311	n/a	writer.pos = pos;
5312	n/a	}
5313	n/a
5314	n/a	if (Py_UNICODE_IS_SURROGATE(ch)) {
5315	n/a	errmsg = "code point in surrogate code point range(0xd800, 0xe000)";
5316	n/a	startinpos = ((const char *)q) - starts;
5317	n/a	endinpos = startinpos + 4;
5318	n/a	}
5319	n/a	else if (ch <= maxch) {
5320	n/a	if (q == e \|\| consumed)
5321	n/a	break;
5322	n/a	/* remaining bytes at the end? (size should be divisible by 4) */
5323	n/a	errmsg = "truncated data";
5324	n/a	startinpos = ((const char *)q) - starts;
5325	n/a	endinpos = ((const char *)e) - starts;
5326	n/a	}
5327	n/a	else {
5328	n/a	if (ch < 0x110000) {
5329	n/a	if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0)
5330	n/a	goto onError;
5331	n/a	q += 4;
5332	n/a	continue;
5333	n/a	}
5334	n/a	errmsg = "code point not in range(0x110000)";
5335	n/a	startinpos = ((const char *)q) - starts;
5336	n/a	endinpos = startinpos + 4;
5337	n/a	}
5338	n/a
5339	n/a	/* The remaining input chars are ignored if the callback
5340	n/a	chooses to skip the input */
5341	n/a	if (unicode_decode_call_errorhandler_writer(
5342	n/a	errors, &errorHandler,
5343	n/a	encoding, errmsg,
5344	n/a	&starts, (const char )&e, &startinpos, &endinpos, &exc, (const char )&q,
5345	n/a	&writer))
5346	n/a	goto onError;
5347	n/a	}
5348	n/a
5349	n/a	if (consumed)
5350	n/a	consumed = (const char )q-starts;
5351	n/a
5352	n/a	Py_XDECREF(errorHandler);
5353	n/a	Py_XDECREF(exc);
5354	n/a	return _PyUnicodeWriter_Finish(&writer);
5355	n/a
5356	n/a	onError:
5357	n/a	_PyUnicodeWriter_Dealloc(&writer);
5358	n/a	Py_XDECREF(errorHandler);
5359	n/a	Py_XDECREF(exc);
5360	n/a	return NULL;
5361	n/a	}
5362	n/a
5363	n/a	PyObject *
5364	n/a	_PyUnicode_EncodeUTF32(PyObject *str,
5365	n/a	const char *errors,
5366	n/a	int byteorder)
5367	n/a	{
5368	n/a	enum PyUnicode_Kind kind;
5369	n/a	const void *data;
5370	n/a	Py_ssize_t len;
5371	n/a	PyObject *v;
5372	n/a	uint32_t *out;
5373	n/a	#if PY_LITTLE_ENDIAN
5374	n/a	int native_ordering = byteorder <= 0;
5375	n/a	#else
5376	n/a	int native_ordering = byteorder >= 0;
5377	n/a	#endif
5378	n/a	const char *encoding;
5379	n/a	Py_ssize_t nsize, pos;
5380	n/a	PyObject *errorHandler = NULL;
5381	n/a	PyObject *exc = NULL;
5382	n/a	PyObject *rep = NULL;
5383	n/a
5384	n/a	if (!PyUnicode_Check(str)) {
5385	n/a	PyErr_BadArgument();
5386	n/a	return NULL;
5387	n/a	}
5388	n/a	if (PyUnicode_READY(str) == -1)
5389	n/a	return NULL;
5390	n/a	kind = PyUnicode_KIND(str);
5391	n/a	data = PyUnicode_DATA(str);
5392	n/a	len = PyUnicode_GET_LENGTH(str);
5393	n/a
5394	n/a	if (len > PY_SSIZE_T_MAX / 4 - (byteorder == 0))
5395	n/a	return PyErr_NoMemory();
5396	n/a	nsize = len + (byteorder == 0);
5397	n/a	v = PyBytes_FromStringAndSize(NULL, nsize * 4);
5398	n/a	if (v == NULL)
5399	n/a	return NULL;
5400	n/a
5401	n/a	/* output buffer is 4-bytes aligned */
5402	n/a	assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(v), 4));
5403	n/a	out = (uint32_t *)PyBytes_AS_STRING(v);
5404	n/a	if (byteorder == 0)
5405	n/a	*out++ = 0xFEFF;
5406	n/a	if (len == 0)
5407	n/a	goto done;
5408	n/a
5409	n/a	if (byteorder == -1)
5410	n/a	encoding = "utf-32-le";
5411	n/a	else if (byteorder == 1)
5412	n/a	encoding = "utf-32-be";
5413	n/a	else
5414	n/a	encoding = "utf-32";
5415	n/a
5416	n/a	if (kind == PyUnicode_1BYTE_KIND) {
5417	n/a	ucs1lib_utf32_encode((const Py_UCS1 *)data, len, &out, native_ordering);
5418	n/a	goto done;
5419	n/a	}
5420	n/a
5421	n/a	pos = 0;
5422	n/a	while (pos < len) {
5423	n/a	Py_ssize_t repsize, moreunits;
5424	n/a
5425	n/a	if (kind == PyUnicode_2BYTE_KIND) {
5426	n/a	pos += ucs2lib_utf32_encode((const Py_UCS2 *)data + pos, len - pos,
5427	n/a	&out, native_ordering);
5428	n/a	}
5429	n/a	else {
5430	n/a	assert(kind == PyUnicode_4BYTE_KIND);
5431	n/a	pos += ucs4lib_utf32_encode((const Py_UCS4 *)data + pos, len - pos,
5432	n/a	&out, native_ordering);
5433	n/a	}
5434	n/a	if (pos == len)
5435	n/a	break;
5436	n/a
5437	n/a	rep = unicode_encode_call_errorhandler(
5438	n/a	errors, &errorHandler,
5439	n/a	encoding, "surrogates not allowed",
5440	n/a	str, &exc, pos, pos + 1, &pos);
5441	n/a	if (!rep)
5442	n/a	goto error;
5443	n/a
5444	n/a	if (PyBytes_Check(rep)) {
5445	n/a	repsize = PyBytes_GET_SIZE(rep);
5446	n/a	if (repsize & 3) {
5447	n/a	raise_encode_exception(&exc, encoding,
5448	n/a	str, pos - 1, pos,
5449	n/a	"surrogates not allowed");
5450	n/a	goto error;
5451	n/a	}
5452	n/a	moreunits = repsize / 4;
5453	n/a	}
5454	n/a	else {
5455	n/a	assert(PyUnicode_Check(rep));
5456	n/a	if (PyUnicode_READY(rep) < 0)
5457	n/a	goto error;
5458	n/a	moreunits = repsize = PyUnicode_GET_LENGTH(rep);
5459	n/a	if (!PyUnicode_IS_ASCII(rep)) {
5460	n/a	raise_encode_exception(&exc, encoding,
5461	n/a	str, pos - 1, pos,
5462	n/a	"surrogates not allowed");
5463	n/a	goto error;
5464	n/a	}
5465	n/a	}
5466	n/a
5467	n/a	/* four bytes are reserved for each surrogate */
5468	n/a	if (moreunits > 1) {
5469	n/a	Py_ssize_t outpos = out - (uint32_t*) PyBytes_AS_STRING(v);
5470	n/a	Py_ssize_t morebytes = 4 * (moreunits - 1);
5471	n/a	if (PyBytes_GET_SIZE(v) > PY_SSIZE_T_MAX - morebytes) {
5472	n/a	/* integer overflow */
5473	n/a	PyErr_NoMemory();
5474	n/a	goto error;
5475	n/a	}
5476	n/a	if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + morebytes) < 0)
5477	n/a	goto error;
5478	n/a	out = (uint32_t*) PyBytes_AS_STRING(v) + outpos;
5479	n/a	}
5480	n/a
5481	n/a	if (PyBytes_Check(rep)) {
5482	n/a	memcpy(out, PyBytes_AS_STRING(rep), repsize);
5483	n/a	out += moreunits;
5484	n/a	} else /* rep is unicode */ {
5485	n/a	assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND);
5486	n/a	ucs1lib_utf32_encode(PyUnicode_1BYTE_DATA(rep), repsize,
5487	n/a	&out, native_ordering);
5488	n/a	}
5489	n/a
5490	n/a	Py_CLEAR(rep);
5491	n/a	}
5492	n/a
5493	n/a	/* Cut back to size actually needed. This is necessary for, for example,
5494	n/a	encoding of a string containing isolated surrogates and the 'ignore'
5495	n/a	handler is used. */
5496	n/a	nsize = (unsigned char) out - (unsigned char) PyBytes_AS_STRING(v);
5497	n/a	if (nsize != PyBytes_GET_SIZE(v))
5498	n/a	_PyBytes_Resize(&v, nsize);
5499	n/a	Py_XDECREF(errorHandler);
5500	n/a	Py_XDECREF(exc);
5501	n/a	done:
5502	n/a	return v;
5503	n/a	error:
5504	n/a	Py_XDECREF(rep);
5505	n/a	Py_XDECREF(errorHandler);
5506	n/a	Py_XDECREF(exc);
5507	n/a	Py_XDECREF(v);
5508	n/a	return NULL;
5509	n/a	}
5510	n/a
5511	n/a	PyObject *
5512	n/a	PyUnicode_EncodeUTF32(const Py_UNICODE *s,
5513	n/a	Py_ssize_t size,
5514	n/a	const char *errors,
5515	n/a	int byteorder)
5516	n/a	{
5517	n/a	PyObject *result;
5518	n/a	PyObject *tmp = PyUnicode_FromWideChar(s, size);
5519	n/a	if (tmp == NULL)
5520	n/a	return NULL;
5521	n/a	result = _PyUnicode_EncodeUTF32(tmp, errors, byteorder);
5522	n/a	Py_DECREF(tmp);
5523	n/a	return result;
5524	n/a	}
5525	n/a
5526	n/a	PyObject *
5527	n/a	PyUnicode_AsUTF32String(PyObject *unicode)
5528	n/a	{
5529	n/a	return _PyUnicode_EncodeUTF32(unicode, NULL, 0);
5530	n/a	}
5531	n/a
5532	n/a	/* --- UTF-16 Codec ------------------------------------------------------- */
5533	n/a
5534	n/a	PyObject *
5535	n/a	PyUnicode_DecodeUTF16(const char *s,
5536	n/a	Py_ssize_t size,
5537	n/a	const char *errors,
5538	n/a	int *byteorder)
5539	n/a	{
5540	n/a	return PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder, NULL);
5541	n/a	}
5542	n/a
5543	n/a	PyObject *
5544	n/a	PyUnicode_DecodeUTF16Stateful(const char *s,
5545	n/a	Py_ssize_t size,
5546	n/a	const char *errors,
5547	n/a	int *byteorder,
5548	n/a	Py_ssize_t *consumed)
5549	n/a	{
5550	n/a	const char *starts = s;
5551	n/a	Py_ssize_t startinpos;
5552	n/a	Py_ssize_t endinpos;
5553	n/a	_PyUnicodeWriter writer;
5554	n/a	const unsigned char q, e;
5555	n/a	int bo = 0; /* assume native ordering by default */
5556	n/a	int native_ordering;
5557	n/a	const char *errmsg = "";
5558	n/a	PyObject *errorHandler = NULL;
5559	n/a	PyObject *exc = NULL;
5560	n/a	const char *encoding;
5561	n/a
5562	n/a	q = (unsigned char *)s;
5563	n/a	e = q + size;
5564	n/a
5565	n/a	if (byteorder)
5566	n/a	bo = *byteorder;
5567	n/a
5568	n/a	/* Check for BOM marks (U+FEFF) in the input and adjust current
5569	n/a	byte order setting accordingly. In native mode, the leading BOM
5570	n/a	mark is skipped, in all other modes, it is copied to the output
5571	n/a	stream as-is (giving a ZWNBSP character). */
5572	n/a	if (bo == 0 && size >= 2) {
5573	n/a	const Py_UCS4 bom = (q[1] << 8) \| q[0];
5574	n/a	if (bom == 0xFEFF) {
5575	n/a	q += 2;
5576	n/a	bo = -1;
5577	n/a	}
5578	n/a	else if (bom == 0xFFFE) {
5579	n/a	q += 2;
5580	n/a	bo = 1;
5581	n/a	}
5582	n/a	if (byteorder)
5583	n/a	*byteorder = bo;
5584	n/a	}
5585	n/a
5586	n/a	if (q == e) {
5587	n/a	if (consumed)
5588	n/a	*consumed = size;
5589	n/a	_Py_RETURN_UNICODE_EMPTY();
5590	n/a	}
5591	n/a
5592	n/a	#if PY_LITTLE_ENDIAN
5593	n/a	native_ordering = bo <= 0;
5594	n/a	encoding = bo <= 0 ? "utf-16-le" : "utf-16-be";
5595	n/a	#else
5596	n/a	native_ordering = bo >= 0;
5597	n/a	encoding = bo >= 0 ? "utf-16-be" : "utf-16-le";
5598	n/a	#endif
5599	n/a
5600	n/a	/* Note: size will always be longer than the resulting Unicode
5601	n/a	character count */
5602	n/a	_PyUnicodeWriter_Init(&writer);
5603	n/a	writer.min_length = (e - q + 1) / 2;
5604	n/a	if (_PyUnicodeWriter_Prepare(&writer, writer.min_length, 127) == -1)
5605	n/a	goto onError;
5606	n/a
5607	n/a	while (1) {
5608	n/a	Py_UCS4 ch = 0;
5609	n/a	if (e - q >= 2) {
5610	n/a	int kind = writer.kind;
5611	n/a	if (kind == PyUnicode_1BYTE_KIND) {
5612	n/a	if (PyUnicode_IS_ASCII(writer.buffer))
5613	n/a	ch = asciilib_utf16_decode(&q, e,
5614	n/a	(Py_UCS1*)writer.data, &writer.pos,
5615	n/a	native_ordering);
5616	n/a	else
5617	n/a	ch = ucs1lib_utf16_decode(&q, e,
5618	n/a	(Py_UCS1*)writer.data, &writer.pos,
5619	n/a	native_ordering);
5620	n/a	} else if (kind == PyUnicode_2BYTE_KIND) {
5621	n/a	ch = ucs2lib_utf16_decode(&q, e,
5622	n/a	(Py_UCS2*)writer.data, &writer.pos,
5623	n/a	native_ordering);
5624	n/a	} else {
5625	n/a	assert(kind == PyUnicode_4BYTE_KIND);
5626	n/a	ch = ucs4lib_utf16_decode(&q, e,
5627	n/a	(Py_UCS4*)writer.data, &writer.pos,
5628	n/a	native_ordering);
5629	n/a	}
5630	n/a	}
5631	n/a
5632	n/a	switch (ch)
5633	n/a	{
5634	n/a	case 0:
5635	n/a	/* remaining byte at the end? (size should be even) */
5636	n/a	if (q == e \|\| consumed)
5637	n/a	goto End;
5638	n/a	errmsg = "truncated data";
5639	n/a	startinpos = ((const char *)q) - starts;
5640	n/a	endinpos = ((const char *)e) - starts;
5641	n/a	break;
5642	n/a	/* The remaining input chars are ignored if the callback
5643	n/a	chooses to skip the input */
5644	n/a	case 1:
5645	n/a	q -= 2;
5646	n/a	if (consumed)
5647	n/a	goto End;
5648	n/a	errmsg = "unexpected end of data";
5649	n/a	startinpos = ((const char *)q) - starts;
5650	n/a	endinpos = ((const char *)e) - starts;
5651	n/a	break;
5652	n/a	case 2:
5653	n/a	errmsg = "illegal encoding";
5654	n/a	startinpos = ((const char *)q) - 2 - starts;
5655	n/a	endinpos = startinpos + 2;
5656	n/a	break;
5657	n/a	case 3:
5658	n/a	errmsg = "illegal UTF-16 surrogate";
5659	n/a	startinpos = ((const char *)q) - 4 - starts;
5660	n/a	endinpos = startinpos + 2;
5661	n/a	break;
5662	n/a	default:
5663	n/a	if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0)
5664	n/a	goto onError;
5665	n/a	continue;
5666	n/a	}
5667	n/a
5668	n/a	if (unicode_decode_call_errorhandler_writer(
5669	n/a	errors,
5670	n/a	&errorHandler,
5671	n/a	encoding, errmsg,
5672	n/a	&starts,
5673	n/a	(const char **)&e,
5674	n/a	&startinpos,
5675	n/a	&endinpos,
5676	n/a	&exc,
5677	n/a	(const char **)&q,
5678	n/a	&writer))
5679	n/a	goto onError;
5680	n/a	}
5681	n/a
5682	n/a	End:
5683	n/a	if (consumed)
5684	n/a	consumed = (const char )q-starts;
5685	n/a
5686	n/a	Py_XDECREF(errorHandler);
5687	n/a	Py_XDECREF(exc);
5688	n/a	return _PyUnicodeWriter_Finish(&writer);
5689	n/a
5690	n/a	onError:
5691	n/a	_PyUnicodeWriter_Dealloc(&writer);
5692	n/a	Py_XDECREF(errorHandler);
5693	n/a	Py_XDECREF(exc);
5694	n/a	return NULL;
5695	n/a	}
5696	n/a
5697	n/a	PyObject *
5698	n/a	_PyUnicode_EncodeUTF16(PyObject *str,
5699	n/a	const char *errors,
5700	n/a	int byteorder)
5701	n/a	{
5702	n/a	enum PyUnicode_Kind kind;
5703	n/a	const void *data;
5704	n/a	Py_ssize_t len;
5705	n/a	PyObject *v;
5706	n/a	unsigned short *out;
5707	n/a	Py_ssize_t pairs;
5708	n/a	#if PY_BIG_ENDIAN
5709	n/a	int native_ordering = byteorder >= 0;
5710	n/a	#else
5711	n/a	int native_ordering = byteorder <= 0;
5712	n/a	#endif
5713	n/a	const char *encoding;
5714	n/a	Py_ssize_t nsize, pos;
5715	n/a	PyObject *errorHandler = NULL;
5716	n/a	PyObject *exc = NULL;
5717	n/a	PyObject *rep = NULL;
5718	n/a
5719	n/a	if (!PyUnicode_Check(str)) {
5720	n/a	PyErr_BadArgument();
5721	n/a	return NULL;
5722	n/a	}
5723	n/a	if (PyUnicode_READY(str) == -1)
5724	n/a	return NULL;
5725	n/a	kind = PyUnicode_KIND(str);
5726	n/a	data = PyUnicode_DATA(str);
5727	n/a	len = PyUnicode_GET_LENGTH(str);
5728	n/a
5729	n/a	pairs = 0;
5730	n/a	if (kind == PyUnicode_4BYTE_KIND) {
5731	n/a	const Py_UCS4 in = (const Py_UCS4 )data;
5732	n/a	const Py_UCS4 *end = in + len;
5733	n/a	while (in < end) {
5734	n/a	if (*in++ >= 0x10000) {
5735	n/a	pairs++;
5736	n/a	}
5737	n/a	}
5738	n/a	}
5739	n/a	if (len > PY_SSIZE_T_MAX / 2 - pairs - (byteorder == 0)) {
5740	n/a	return PyErr_NoMemory();
5741	n/a	}
5742	n/a	nsize = len + pairs + (byteorder == 0);
5743	n/a	v = PyBytes_FromStringAndSize(NULL, nsize * 2);
5744	n/a	if (v == NULL) {
5745	n/a	return NULL;
5746	n/a	}
5747	n/a
5748	n/a	/* output buffer is 2-bytes aligned */
5749	n/a	assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(v), 2));
5750	n/a	out = (unsigned short *)PyBytes_AS_STRING(v);
5751	n/a	if (byteorder == 0) {
5752	n/a	*out++ = 0xFEFF;
5753	n/a	}
5754	n/a	if (len == 0) {
5755	n/a	goto done;
5756	n/a	}
5757	n/a
5758	n/a	if (kind == PyUnicode_1BYTE_KIND) {
5759	n/a	ucs1lib_utf16_encode((const Py_UCS1 *)data, len, &out, native_ordering);
5760	n/a	goto done;
5761	n/a	}
5762	n/a
5763	n/a	if (byteorder < 0) {
5764	n/a	encoding = "utf-16-le";
5765	n/a	}
5766	n/a	else if (byteorder > 0) {
5767	n/a	encoding = "utf-16-be";
5768	n/a	}
5769	n/a	else {
5770	n/a	encoding = "utf-16";
5771	n/a	}
5772	n/a
5773	n/a	pos = 0;
5774	n/a	while (pos < len) {
5775	n/a	Py_ssize_t repsize, moreunits;
5776	n/a
5777	n/a	if (kind == PyUnicode_2BYTE_KIND) {
5778	n/a	pos += ucs2lib_utf16_encode((const Py_UCS2 *)data + pos, len - pos,
5779	n/a	&out, native_ordering);
5780	n/a	}
5781	n/a	else {
5782	n/a	assert(kind == PyUnicode_4BYTE_KIND);
5783	n/a	pos += ucs4lib_utf16_encode((const Py_UCS4 *)data + pos, len - pos,
5784	n/a	&out, native_ordering);
5785	n/a	}
5786	n/a	if (pos == len)
5787	n/a	break;
5788	n/a
5789	n/a	rep = unicode_encode_call_errorhandler(
5790	n/a	errors, &errorHandler,
5791	n/a	encoding, "surrogates not allowed",
5792	n/a	str, &exc, pos, pos + 1, &pos);
5793	n/a	if (!rep)
5794	n/a	goto error;
5795	n/a
5796	n/a	if (PyBytes_Check(rep)) {
5797	n/a	repsize = PyBytes_GET_SIZE(rep);
5798	n/a	if (repsize & 1) {
5799	n/a	raise_encode_exception(&exc, encoding,
5800	n/a	str, pos - 1, pos,
5801	n/a	"surrogates not allowed");
5802	n/a	goto error;
5803	n/a	}
5804	n/a	moreunits = repsize / 2;
5805	n/a	}
5806	n/a	else {
5807	n/a	assert(PyUnicode_Check(rep));
5808	n/a	if (PyUnicode_READY(rep) < 0)
5809	n/a	goto error;
5810	n/a	moreunits = repsize = PyUnicode_GET_LENGTH(rep);
5811	n/a	if (!PyUnicode_IS_ASCII(rep)) {
5812	n/a	raise_encode_exception(&exc, encoding,
5813	n/a	str, pos - 1, pos,
5814	n/a	"surrogates not allowed");
5815	n/a	goto error;
5816	n/a	}
5817	n/a	}
5818	n/a
5819	n/a	/* two bytes are reserved for each surrogate */
5820	n/a	if (moreunits > 1) {
5821	n/a	Py_ssize_t outpos = out - (unsigned short*) PyBytes_AS_STRING(v);
5822	n/a	Py_ssize_t morebytes = 2 * (moreunits - 1);
5823	n/a	if (PyBytes_GET_SIZE(v) > PY_SSIZE_T_MAX - morebytes) {
5824	n/a	/* integer overflow */
5825	n/a	PyErr_NoMemory();
5826	n/a	goto error;
5827	n/a	}
5828	n/a	if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + morebytes) < 0)
5829	n/a	goto error;
5830	n/a	out = (unsigned short*) PyBytes_AS_STRING(v) + outpos;
5831	n/a	}
5832	n/a
5833	n/a	if (PyBytes_Check(rep)) {
5834	n/a	memcpy(out, PyBytes_AS_STRING(rep), repsize);
5835	n/a	out += moreunits;
5836	n/a	} else /* rep is unicode */ {
5837	n/a	assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND);
5838	n/a	ucs1lib_utf16_encode(PyUnicode_1BYTE_DATA(rep), repsize,
5839	n/a	&out, native_ordering);
5840	n/a	}
5841	n/a
5842	n/a	Py_CLEAR(rep);
5843	n/a	}
5844	n/a
5845	n/a	/* Cut back to size actually needed. This is necessary for, for example,
5846	n/a	encoding of a string containing isolated surrogates and the 'ignore' handler
5847	n/a	is used. */
5848	n/a	nsize = (unsigned char) out - (unsigned char) PyBytes_AS_STRING(v);
5849	n/a	if (nsize != PyBytes_GET_SIZE(v))
5850	n/a	_PyBytes_Resize(&v, nsize);
5851	n/a	Py_XDECREF(errorHandler);
5852	n/a	Py_XDECREF(exc);
5853	n/a	done:
5854	n/a	return v;
5855	n/a	error:
5856	n/a	Py_XDECREF(rep);
5857	n/a	Py_XDECREF(errorHandler);
5858	n/a	Py_XDECREF(exc);
5859	n/a	Py_XDECREF(v);
5860	n/a	return NULL;
5861	n/a	#undef STORECHAR
5862	n/a	}
5863	n/a
5864	n/a	PyObject *
5865	n/a	PyUnicode_EncodeUTF16(const Py_UNICODE *s,
5866	n/a	Py_ssize_t size,
5867	n/a	const char *errors,
5868	n/a	int byteorder)
5869	n/a	{
5870	n/a	PyObject *result;
5871	n/a	PyObject *tmp = PyUnicode_FromWideChar(s, size);
5872	n/a	if (tmp == NULL)
5873	n/a	return NULL;
5874	n/a	result = _PyUnicode_EncodeUTF16(tmp, errors, byteorder);
5875	n/a	Py_DECREF(tmp);
5876	n/a	return result;
5877	n/a	}
5878	n/a
5879	n/a	PyObject *
5880	n/a	PyUnicode_AsUTF16String(PyObject *unicode)
5881	n/a	{
5882	n/a	return _PyUnicode_EncodeUTF16(unicode, NULL, 0);
5883	n/a	}
5884	n/a
5885	n/a	/* --- Unicode Escape Codec ----------------------------------------------- */
5886	n/a
5887	n/a	static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL;
5888	n/a
5889	n/a	PyObject *
5890	n/a	_PyUnicode_DecodeUnicodeEscape(const char *s,
5891	n/a	Py_ssize_t size,
5892	n/a	const char *errors,
5893	n/a	const char **first_invalid_escape)
5894	n/a	{
5895	n/a	const char *starts = s;
5896	n/a	_PyUnicodeWriter writer;
5897	n/a	const char *end;
5898	n/a	PyObject *errorHandler = NULL;
5899	n/a	PyObject *exc = NULL;
5900	n/a
5901	n/a	// so we can remember if we've seen an invalid escape char or not
5902	n/a	*first_invalid_escape = NULL;
5903	n/a
5904	n/a	if (size == 0) {
5905	n/a	_Py_RETURN_UNICODE_EMPTY();
5906	n/a	}
5907	n/a	/* Escaped strings will always be longer than the resulting
5908	n/a	Unicode string, so we start with size here and then reduce the
5909	n/a	length after conversion to the true value.
5910	n/a	(but if the error callback returns a long replacement string
5911	n/a	we'll have to allocate more space) */
5912	n/a	_PyUnicodeWriter_Init(&writer);
5913	n/a	writer.min_length = size;
5914	n/a	if (_PyUnicodeWriter_Prepare(&writer, size, 127) < 0) {
5915	n/a	goto onError;
5916	n/a	}
5917	n/a
5918	n/a	end = s + size;
5919	n/a	while (s < end) {
5920	n/a	unsigned char c = (unsigned char) *s++;
5921	n/a	Py_UCS4 ch;
5922	n/a	int count;
5923	n/a	Py_ssize_t startinpos;
5924	n/a	Py_ssize_t endinpos;
5925	n/a	const char *message;
5926	n/a
5927	n/a	#define WRITE_ASCII_CHAR(ch) \
5928	n/a	do { \
5929	n/a	assert(ch <= 127); \
5930	n/a	assert(writer.pos < writer.size); \
5931	n/a	PyUnicode_WRITE(writer.kind, writer.data, writer.pos++, ch); \
5932	n/a	} while(0)
5933	n/a
5934	n/a	#define WRITE_CHAR(ch) \
5935	n/a	do { \
5936	n/a	if (ch <= writer.maxchar) { \
5937	n/a	assert(writer.pos < writer.size); \
5938	n/a	PyUnicode_WRITE(writer.kind, writer.data, writer.pos++, ch); \
5939	n/a	} \
5940	n/a	else if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0) { \
5941	n/a	goto onError; \
5942	n/a	} \
5943	n/a	} while(0)
5944	n/a
5945	n/a	/* Non-escape characters are interpreted as Unicode ordinals */
5946	n/a	if (c != '\\') {
5947	n/a	WRITE_CHAR(c);
5948	n/a	continue;
5949	n/a	}
5950	n/a
5951	n/a	startinpos = s - starts - 1;
5952	n/a	/* \ - Escapes */
5953	n/a	if (s >= end) {
5954	n/a	message = "\\ at end of string";
5955	n/a	goto error;
5956	n/a	}
5957	n/a	c = (unsigned char) *s++;
5958	n/a
5959	n/a	assert(writer.pos < writer.size);
5960	n/a	switch (c) {
5961	n/a
5962	n/a	/* \x escapes */
5963	n/a	case '\n': continue;
5964	n/a	case '\\': WRITE_ASCII_CHAR('\\'); continue;
5965	n/a	case '\'': WRITE_ASCII_CHAR('\''); continue;
5966	n/a	case '\"': WRITE_ASCII_CHAR('\"'); continue;
5967	n/a	case 'b': WRITE_ASCII_CHAR('\b'); continue;
5968	n/a	/* FF */
5969	n/a	case 'f': WRITE_ASCII_CHAR('\014'); continue;
5970	n/a	case 't': WRITE_ASCII_CHAR('\t'); continue;
5971	n/a	case 'n': WRITE_ASCII_CHAR('\n'); continue;
5972	n/a	case 'r': WRITE_ASCII_CHAR('\r'); continue;
5973	n/a	/* VT */
5974	n/a	case 'v': WRITE_ASCII_CHAR('\013'); continue;
5975	n/a	/* BEL, not classic C */
5976	n/a	case 'a': WRITE_ASCII_CHAR('\007'); continue;
5977	n/a
5978	n/a	/* \OOO (octal) escapes */
5979	n/a	case '0': case '1': case '2': case '3':
5980	n/a	case '4': case '5': case '6': case '7':
5981	n/a	ch = c - '0';
5982	n/a	if (s < end && '0' <= s && s <= '7') {
5983	n/a	ch = (ch<<3) + *s++ - '0';
5984	n/a	if (s < end && '0' <= s && s <= '7') {
5985	n/a	ch = (ch<<3) + *s++ - '0';
5986	n/a	}
5987	n/a	}
5988	n/a	WRITE_CHAR(ch);
5989	n/a	continue;
5990	n/a
5991	n/a	/* hex escapes */
5992	n/a	/* \xXX */
5993	n/a	case 'x':
5994	n/a	count = 2;
5995	n/a	message = "truncated \\xXX escape";
5996	n/a	goto hexescape;
5997	n/a
5998	n/a	/* \uXXXX */
5999	n/a	case 'u':
6000	n/a	count = 4;
6001	n/a	message = "truncated \\uXXXX escape";
6002	n/a	goto hexescape;
6003	n/a
6004	n/a	/* \UXXXXXXXX */
6005	n/a	case 'U':
6006	n/a	count = 8;
6007	n/a	message = "truncated \\UXXXXXXXX escape";
6008	n/a	hexescape:
6009	n/a	for (ch = 0; count && s < end; ++s, --count) {
6010	n/a	c = (unsigned char)*s;
6011	n/a	ch <<= 4;
6012	n/a	if (c >= '0' && c <= '9') {
6013	n/a	ch += c - '0';
6014	n/a	}
6015	n/a	else if (c >= 'a' && c <= 'f') {
6016	n/a	ch += c - ('a' - 10);
6017	n/a	}
6018	n/a	else if (c >= 'A' && c <= 'F') {
6019	n/a	ch += c - ('A' - 10);
6020	n/a	}
6021	n/a	else {
6022	n/a	break;
6023	n/a	}
6024	n/a	}
6025	n/a	if (count) {
6026	n/a	goto error;
6027	n/a	}
6028	n/a
6029	n/a	/* when we get here, ch is a 32-bit unicode character */
6030	n/a	if (ch > MAX_UNICODE) {
6031	n/a	message = "illegal Unicode character";
6032	n/a	goto error;
6033	n/a	}
6034	n/a
6035	n/a	WRITE_CHAR(ch);
6036	n/a	continue;
6037	n/a
6038	n/a	/* \N{name} */
6039	n/a	case 'N':
6040	n/a	if (ucnhash_CAPI == NULL) {
6041	n/a	/* load the unicode data module */
6042	n/a	ucnhash_CAPI = (_PyUnicode_Name_CAPI *)PyCapsule_Import(
6043	n/a	PyUnicodeData_CAPSULE_NAME, 1);
6044	n/a	if (ucnhash_CAPI == NULL) {
6045	n/a	PyErr_SetString(
6046	n/a	PyExc_UnicodeError,
6047	n/a	"\\N escapes not supported (can't load unicodedata module)"
6048	n/a	);
6049	n/a	goto onError;
6050	n/a	}
6051	n/a	}
6052	n/a
6053	n/a	message = "malformed \\N character escape";
6054	n/a	if (*s == '{') {
6055	n/a	const char *start = ++s;
6056	n/a	size_t namelen;
6057	n/a	/* look for the closing brace */
6058	n/a	while (s < end && *s != '}')
6059	n/a	s++;
6060	n/a	namelen = s - start;
6061	n/a	if (namelen && s < end) {
6062	n/a	/* found a name. look it up in the unicode database */
6063	n/a	s++;
6064	n/a	ch = 0xffffffff; /* in case 'getcode' messes up */
6065	n/a	if (namelen <= INT_MAX &&
6066	n/a	ucnhash_CAPI->getcode(NULL, start, (int)namelen,
6067	n/a	&ch, 0)) {
6068	n/a	assert(ch <= MAX_UNICODE);
6069	n/a	WRITE_CHAR(ch);
6070	n/a	continue;
6071	n/a	}
6072	n/a	message = "unknown Unicode character name";
6073	n/a	}
6074	n/a	}
6075	n/a	goto error;
6076	n/a
6077	n/a	default:
6078	n/a	if (*first_invalid_escape == NULL) {
6079	n/a	first_invalid_escape = s-1; / Back up one char, since we've
6080	n/a	already incremented s. */
6081	n/a	}
6082	n/a	WRITE_ASCII_CHAR('\\');
6083	n/a	WRITE_CHAR(c);
6084	n/a	continue;
6085	n/a	}
6086	n/a
6087	n/a	error:
6088	n/a	endinpos = s-starts;
6089	n/a	writer.min_length = end - s + writer.pos;
6090	n/a	if (unicode_decode_call_errorhandler_writer(
6091	n/a	errors, &errorHandler,
6092	n/a	"unicodeescape", message,
6093	n/a	&starts, &end, &startinpos, &endinpos, &exc, &s,
6094	n/a	&writer)) {
6095	n/a	goto onError;
6096	n/a	}
6097	n/a	if (_PyUnicodeWriter_Prepare(&writer, writer.min_length, 127) < 0) {
6098	n/a	goto onError;
6099	n/a	}
6100	n/a
6101	n/a	#undef WRITE_ASCII_CHAR
6102	n/a	#undef WRITE_CHAR
6103	n/a	}
6104	n/a
6105	n/a	Py_XDECREF(errorHandler);
6106	n/a	Py_XDECREF(exc);
6107	n/a	return _PyUnicodeWriter_Finish(&writer);
6108	n/a
6109	n/a	onError:
6110	n/a	_PyUnicodeWriter_Dealloc(&writer);
6111	n/a	Py_XDECREF(errorHandler);
6112	n/a	Py_XDECREF(exc);
6113	n/a	return NULL;
6114	n/a	}
6115	n/a
6116	n/a	PyObject *
6117	n/a	PyUnicode_DecodeUnicodeEscape(const char *s,
6118	n/a	Py_ssize_t size,
6119	n/a	const char *errors)
6120	n/a	{
6121	n/a	const char *first_invalid_escape;
6122	n/a	PyObject *result = _PyUnicode_DecodeUnicodeEscape(s, size, errors,
6123	n/a	&first_invalid_escape);
6124	n/a	if (result == NULL)
6125	n/a	return NULL;
6126	n/a	if (first_invalid_escape != NULL) {
6127	n/a	if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
6128	n/a	"invalid escape sequence '\\%c'",
6129	n/a	*first_invalid_escape) < 0) {
6130	n/a	Py_DECREF(result);
6131	n/a	return NULL;
6132	n/a	}
6133	n/a	}
6134	n/a	return result;
6135	n/a	}
6136	n/a
6137	n/a	/* Return a Unicode-Escape string version of the Unicode object. */
6138	n/a
6139	n/a	PyObject *
6140	n/a	PyUnicode_AsUnicodeEscapeString(PyObject *unicode)
6141	n/a	{
6142	n/a	Py_ssize_t i, len;
6143	n/a	PyObject *repr;
6144	n/a	char *p;
6145	n/a	enum PyUnicode_Kind kind;
6146	n/a	void *data;
6147	n/a	Py_ssize_t expandsize;
6148	n/a
6149	n/a	/* Initial allocation is based on the longest-possible character
6150	n/a	escape.
6151	n/a
6152	n/a	For UCS1 strings it's '\xxx', 4 bytes per source character.
6153	n/a	For UCS2 strings it's '\uxxxx', 6 bytes per source character.
6154	n/a	For UCS4 strings it's '\U00xxxxxx', 10 bytes per source character.
6155	n/a	*/
6156	n/a
6157	n/a	if (!PyUnicode_Check(unicode)) {
6158	n/a	PyErr_BadArgument();
6159	n/a	return NULL;
6160	n/a	}
6161	n/a	if (PyUnicode_READY(unicode) == -1) {
6162	n/a	return NULL;
6163	n/a	}
6164	n/a
6165	n/a	len = PyUnicode_GET_LENGTH(unicode);
6166	n/a	if (len == 0) {
6167	n/a	return PyBytes_FromStringAndSize(NULL, 0);
6168	n/a	}
6169	n/a
6170	n/a	kind = PyUnicode_KIND(unicode);
6171	n/a	data = PyUnicode_DATA(unicode);
6172	n/a	/* 4 byte characters can take up 10 bytes, 2 byte characters can take up 6
6173	n/a	bytes, and 1 byte characters 4. */
6174	n/a	expandsize = kind * 2 + 2;
6175	n/a	if (len > PY_SSIZE_T_MAX / expandsize) {
6176	n/a	return PyErr_NoMemory();
6177	n/a	}
6178	n/a	repr = PyBytes_FromStringAndSize(NULL, expandsize * len);
6179	n/a	if (repr == NULL) {
6180	n/a	return NULL;
6181	n/a	}
6182	n/a
6183	n/a	p = PyBytes_AS_STRING(repr);
6184	n/a	for (i = 0; i < len; i++) {
6185	n/a	Py_UCS4 ch = PyUnicode_READ(kind, data, i);
6186	n/a
6187	n/a	/* U+0000-U+00ff range */
6188	n/a	if (ch < 0x100) {
6189	n/a	if (ch >= ' ' && ch < 127) {
6190	n/a	if (ch != '\\') {
6191	n/a	/* Copy printable US ASCII as-is */
6192	n/a	*p++ = (char) ch;
6193	n/a	}
6194	n/a	/* Escape backslashes */
6195	n/a	else {
6196	n/a	*p++ = '\\';
6197	n/a	*p++ = '\\';
6198	n/a	}
6199	n/a	}
6200	n/a
6201	n/a	/* Map special whitespace to '\t', \n', '\r' */
6202	n/a	else if (ch == '\t') {
6203	n/a	*p++ = '\\';
6204	n/a	*p++ = 't';
6205	n/a	}
6206	n/a	else if (ch == '\n') {
6207	n/a	*p++ = '\\';
6208	n/a	*p++ = 'n';
6209	n/a	}
6210	n/a	else if (ch == '\r') {
6211	n/a	*p++ = '\\';
6212	n/a	*p++ = 'r';
6213	n/a	}
6214	n/a
6215	n/a	/* Map non-printable US ASCII and 8-bit characters to '\xHH' */
6216	n/a	else {
6217	n/a	*p++ = '\\';
6218	n/a	*p++ = 'x';
6219	n/a	*p++ = Py_hexdigits[(ch >> 4) & 0x000F];
6220	n/a	*p++ = Py_hexdigits[ch & 0x000F];
6221	n/a	}
6222	n/a	}
6223	n/a	/* U+0100-U+ffff range: Map 16-bit characters to '\uHHHH' */
6224	n/a	else if (ch < 0x10000) {
6225	n/a	*p++ = '\\';
6226	n/a	*p++ = 'u';
6227	n/a	*p++ = Py_hexdigits[(ch >> 12) & 0x000F];
6228	n/a	*p++ = Py_hexdigits[(ch >> 8) & 0x000F];
6229	n/a	*p++ = Py_hexdigits[(ch >> 4) & 0x000F];
6230	n/a	*p++ = Py_hexdigits[ch & 0x000F];
6231	n/a	}
6232	n/a	/* U+010000-U+10ffff range: Map 21-bit characters to '\U00HHHHHH' */
6233	n/a	else {
6234	n/a
6235	n/a	/* Make sure that the first two digits are zero */
6236	n/a	assert(ch <= MAX_UNICODE && MAX_UNICODE <= 0x10ffff);
6237	n/a	*p++ = '\\';
6238	n/a	*p++ = 'U';
6239	n/a	*p++ = '0';
6240	n/a	*p++ = '0';
6241	n/a	*p++ = Py_hexdigits[(ch >> 20) & 0x0000000F];
6242	n/a	*p++ = Py_hexdigits[(ch >> 16) & 0x0000000F];
6243	n/a	*p++ = Py_hexdigits[(ch >> 12) & 0x0000000F];
6244	n/a	*p++ = Py_hexdigits[(ch >> 8) & 0x0000000F];
6245	n/a	*p++ = Py_hexdigits[(ch >> 4) & 0x0000000F];
6246	n/a	*p++ = Py_hexdigits[ch & 0x0000000F];
6247	n/a	}
6248	n/a	}
6249	n/a
6250	n/a	assert(p - PyBytes_AS_STRING(repr) > 0);
6251	n/a	if (_PyBytes_Resize(&repr, p - PyBytes_AS_STRING(repr)) < 0) {
6252	n/a	return NULL;
6253	n/a	}
6254	n/a	return repr;
6255	n/a	}
6256	n/a
6257	n/a	PyObject *
6258	n/a	PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s,
6259	n/a	Py_ssize_t size)
6260	n/a	{
6261	n/a	PyObject *result;
6262	n/a	PyObject *tmp = PyUnicode_FromWideChar(s, size);
6263	n/a	if (tmp == NULL) {
6264	n/a	return NULL;
6265	n/a	}
6266	n/a
6267	n/a	result = PyUnicode_AsUnicodeEscapeString(tmp);
6268	n/a	Py_DECREF(tmp);
6269	n/a	return result;
6270	n/a	}
6271	n/a
6272	n/a	/* --- Raw Unicode Escape Codec ------------------------------------------- */
6273	n/a
6274	n/a	PyObject *
6275	n/a	PyUnicode_DecodeRawUnicodeEscape(const char *s,
6276	n/a	Py_ssize_t size,
6277	n/a	const char *errors)
6278	n/a	{
6279	n/a	const char *starts = s;
6280	n/a	_PyUnicodeWriter writer;
6281	n/a	const char *end;
6282	n/a	PyObject *errorHandler = NULL;
6283	n/a	PyObject *exc = NULL;
6284	n/a
6285	n/a	if (size == 0) {
6286	n/a	_Py_RETURN_UNICODE_EMPTY();
6287	n/a	}
6288	n/a
6289	n/a	/* Escaped strings will always be longer than the resulting
6290	n/a	Unicode string, so we start with size here and then reduce the
6291	n/a	length after conversion to the true value. (But decoding error
6292	n/a	handler might have to resize the string) */
6293	n/a	_PyUnicodeWriter_Init(&writer);
6294	n/a	writer.min_length = size;
6295	n/a	if (_PyUnicodeWriter_Prepare(&writer, size, 127) < 0) {
6296	n/a	goto onError;
6297	n/a	}
6298	n/a
6299	n/a	end = s + size;
6300	n/a	while (s < end) {
6301	n/a	unsigned char c = (unsigned char) *s++;
6302	n/a	Py_UCS4 ch;
6303	n/a	int count;
6304	n/a	Py_ssize_t startinpos;
6305	n/a	Py_ssize_t endinpos;
6306	n/a	const char *message;
6307	n/a
6308	n/a	#define WRITE_CHAR(ch) \
6309	n/a	do { \
6310	n/a	if (ch <= writer.maxchar) { \
6311	n/a	assert(writer.pos < writer.size); \
6312	n/a	PyUnicode_WRITE(writer.kind, writer.data, writer.pos++, ch); \
6313	n/a	} \
6314	n/a	else if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0) { \
6315	n/a	goto onError; \
6316	n/a	} \
6317	n/a	} while(0)
6318	n/a
6319	n/a	/* Non-escape characters are interpreted as Unicode ordinals */
6320	n/a	if (c != '\\' \|\| s >= end) {
6321	n/a	WRITE_CHAR(c);
6322	n/a	continue;
6323	n/a	}
6324	n/a
6325	n/a	c = (unsigned char) *s++;
6326	n/a	if (c == 'u') {
6327	n/a	count = 4;
6328	n/a	message = "truncated \\uXXXX escape";
6329	n/a	}
6330	n/a	else if (c == 'U') {
6331	n/a	count = 8;
6332	n/a	message = "truncated \\UXXXXXXXX escape";
6333	n/a	}
6334	n/a	else {
6335	n/a	assert(writer.pos < writer.size);
6336	n/a	PyUnicode_WRITE(writer.kind, writer.data, writer.pos++, '\\');
6337	n/a	WRITE_CHAR(c);
6338	n/a	continue;
6339	n/a	}
6340	n/a	startinpos = s - starts - 2;
6341	n/a
6342	n/a	/* \uHHHH with 4 hex digits, \U00HHHHHH with 8 */
6343	n/a	for (ch = 0; count && s < end; ++s, --count) {
6344	n/a	c = (unsigned char)*s;
6345	n/a	ch <<= 4;
6346	n/a	if (c >= '0' && c <= '9') {
6347	n/a	ch += c - '0';
6348	n/a	}
6349	n/a	else if (c >= 'a' && c <= 'f') {
6350	n/a	ch += c - ('a' - 10);
6351	n/a	}
6352	n/a	else if (c >= 'A' && c <= 'F') {
6353	n/a	ch += c - ('A' - 10);
6354	n/a	}
6355	n/a	else {
6356	n/a	break;
6357	n/a	}
6358	n/a	}
6359	n/a	if (!count) {
6360	n/a	if (ch <= MAX_UNICODE) {
6361	n/a	WRITE_CHAR(ch);
6362	n/a	continue;
6363	n/a	}
6364	n/a	message = "\\Uxxxxxxxx out of range";
6365	n/a	}
6366	n/a
6367	n/a	endinpos = s-starts;
6368	n/a	writer.min_length = end - s + writer.pos;
6369	n/a	if (unicode_decode_call_errorhandler_writer(
6370	n/a	errors, &errorHandler,
6371	n/a	"rawunicodeescape", message,
6372	n/a	&starts, &end, &startinpos, &endinpos, &exc, &s,
6373	n/a	&writer)) {
6374	n/a	goto onError;
6375	n/a	}
6376	n/a	if (_PyUnicodeWriter_Prepare(&writer, writer.min_length, 127) < 0) {
6377	n/a	goto onError;
6378	n/a	}
6379	n/a
6380	n/a	#undef WRITE_CHAR
6381	n/a	}
6382	n/a	Py_XDECREF(errorHandler);
6383	n/a	Py_XDECREF(exc);
6384	n/a	return _PyUnicodeWriter_Finish(&writer);
6385	n/a
6386	n/a	onError:
6387	n/a	_PyUnicodeWriter_Dealloc(&writer);
6388	n/a	Py_XDECREF(errorHandler);
6389	n/a	Py_XDECREF(exc);
6390	n/a	return NULL;
6391	n/a
6392	n/a	}
6393	n/a
6394	n/a
6395	n/a	PyObject *
6396	n/a	PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode)
6397	n/a	{
6398	n/a	PyObject *repr;
6399	n/a	char *p;
6400	n/a	Py_ssize_t expandsize, pos;
6401	n/a	int kind;
6402	n/a	void *data;
6403	n/a	Py_ssize_t len;
6404	n/a
6405	n/a	if (!PyUnicode_Check(unicode)) {
6406	n/a	PyErr_BadArgument();
6407	n/a	return NULL;
6408	n/a	}
6409	n/a	if (PyUnicode_READY(unicode) == -1) {
6410	n/a	return NULL;
6411	n/a	}
6412	n/a	kind = PyUnicode_KIND(unicode);
6413	n/a	data = PyUnicode_DATA(unicode);
6414	n/a	len = PyUnicode_GET_LENGTH(unicode);
6415	n/a	if (kind == PyUnicode_1BYTE_KIND) {
6416	n/a	return PyBytes_FromStringAndSize(data, len);
6417	n/a	}
6418	n/a
6419	n/a	/* 4 byte characters can take up 10 bytes, 2 byte characters can take up 6
6420	n/a	bytes, and 1 byte characters 4. */
6421	n/a	expandsize = kind * 2 + 2;
6422	n/a
6423	n/a	if (len > PY_SSIZE_T_MAX / expandsize) {
6424	n/a	return PyErr_NoMemory();
6425	n/a	}
6426	n/a	repr = PyBytes_FromStringAndSize(NULL, expandsize * len);
6427	n/a	if (repr == NULL) {
6428	n/a	return NULL;
6429	n/a	}
6430	n/a	if (len == 0) {
6431	n/a	return repr;
6432	n/a	}
6433	n/a
6434	n/a	p = PyBytes_AS_STRING(repr);
6435	n/a	for (pos = 0; pos < len; pos++) {
6436	n/a	Py_UCS4 ch = PyUnicode_READ(kind, data, pos);
6437	n/a
6438	n/a	/* U+0000-U+00ff range: Copy 8-bit characters as-is */
6439	n/a	if (ch < 0x100) {
6440	n/a	*p++ = (char) ch;
6441	n/a	}
6442	n/a	/* U+0000-U+00ff range: Map 16-bit characters to '\uHHHH' */
6443	n/a	else if (ch < 0x10000) {
6444	n/a	*p++ = '\\';
6445	n/a	*p++ = 'u';
6446	n/a	*p++ = Py_hexdigits[(ch >> 12) & 0xf];
6447	n/a	*p++ = Py_hexdigits[(ch >> 8) & 0xf];
6448	n/a	*p++ = Py_hexdigits[(ch >> 4) & 0xf];
6449	n/a	*p++ = Py_hexdigits[ch & 15];
6450	n/a	}
6451	n/a	/* U+010000-U+10ffff range: Map 32-bit characters to '\U00HHHHHH' */
6452	n/a	else {
6453	n/a	assert(ch <= MAX_UNICODE && MAX_UNICODE <= 0x10ffff);
6454	n/a	*p++ = '\\';
6455	n/a	*p++ = 'U';
6456	n/a	*p++ = '0';
6457	n/a	*p++ = '0';
6458	n/a	*p++ = Py_hexdigits[(ch >> 20) & 0xf];
6459	n/a	*p++ = Py_hexdigits[(ch >> 16) & 0xf];
6460	n/a	*p++ = Py_hexdigits[(ch >> 12) & 0xf];
6461	n/a	*p++ = Py_hexdigits[(ch >> 8) & 0xf];
6462	n/a	*p++ = Py_hexdigits[(ch >> 4) & 0xf];
6463	n/a	*p++ = Py_hexdigits[ch & 15];
6464	n/a	}
6465	n/a	}
6466	n/a
6467	n/a	assert(p > PyBytes_AS_STRING(repr));
6468	n/a	if (_PyBytes_Resize(&repr, p - PyBytes_AS_STRING(repr)) < 0) {
6469	n/a	return NULL;
6470	n/a	}
6471	n/a	return repr;
6472	n/a	}
6473	n/a
6474	n/a	PyObject *
6475	n/a	PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s,
6476	n/a	Py_ssize_t size)
6477	n/a	{
6478	n/a	PyObject *result;
6479	n/a	PyObject *tmp = PyUnicode_FromWideChar(s, size);
6480	n/a	if (tmp == NULL)
6481	n/a	return NULL;
6482	n/a	result = PyUnicode_AsRawUnicodeEscapeString(tmp);
6483	n/a	Py_DECREF(tmp);
6484	n/a	return result;
6485	n/a	}
6486	n/a
6487	n/a	/* --- Unicode Internal Codec ------------------------------------------- */
6488	n/a
6489	n/a	PyObject *
6490	n/a	_PyUnicode_DecodeUnicodeInternal(const char *s,
6491	n/a	Py_ssize_t size,
6492	n/a	const char *errors)
6493	n/a	{
6494	n/a	const char *starts = s;
6495	n/a	Py_ssize_t startinpos;
6496	n/a	Py_ssize_t endinpos;
6497	n/a	_PyUnicodeWriter writer;
6498	n/a	const char *end;
6499	n/a	const char *reason;
6500	n/a	PyObject *errorHandler = NULL;
6501	n/a	PyObject *exc = NULL;
6502	n/a
6503	n/a	if (PyErr_WarnEx(PyExc_DeprecationWarning,
6504	n/a	"unicode_internal codec has been deprecated",
6505	n/a	1))
6506	n/a	return NULL;
6507	n/a
6508	n/a	if (size == 0)
6509	n/a	_Py_RETURN_UNICODE_EMPTY();
6510	n/a
6511	n/a	_PyUnicodeWriter_Init(&writer);
6512	n/a	if (size / Py_UNICODE_SIZE > PY_SSIZE_T_MAX - 1) {
6513	n/a	PyErr_NoMemory();
6514	n/a	goto onError;
6515	n/a	}
6516	n/a	writer.min_length = (size + (Py_UNICODE_SIZE - 1)) / Py_UNICODE_SIZE;
6517	n/a
6518	n/a	end = s + size;
6519	n/a	while (s < end) {
6520	n/a	Py_UNICODE uch;
6521	n/a	Py_UCS4 ch;
6522	n/a	if (end - s < Py_UNICODE_SIZE) {
6523	n/a	endinpos = end-starts;
6524	n/a	reason = "truncated input";
6525	n/a	goto error;
6526	n/a	}
6527	n/a	/* We copy the raw representation one byte at a time because the
6528	n/a	pointer may be unaligned (see test_codeccallbacks). */
6529	n/a	((char *) &uch)[0] = s[0];
6530	n/a	((char *) &uch)[1] = s[1];
6531	n/a	#ifdef Py_UNICODE_WIDE
6532	n/a	((char *) &uch)[2] = s[2];
6533	n/a	((char *) &uch)[3] = s[3];
6534	n/a	#endif
6535	n/a	ch = uch;
6536	n/a	#ifdef Py_UNICODE_WIDE
6537	n/a	/* We have to sanity check the raw data, otherwise doom looms for
6538	n/a	some malformed UCS-4 data. */
6539	n/a	if (ch > 0x10ffff) {
6540	n/a	endinpos = s - starts + Py_UNICODE_SIZE;
6541	n/a	reason = "illegal code point (> 0x10FFFF)";
6542	n/a	goto error;
6543	n/a	}
6544	n/a	#endif
6545	n/a	s += Py_UNICODE_SIZE;
6546	n/a	#ifndef Py_UNICODE_WIDE
6547	n/a	if (Py_UNICODE_IS_HIGH_SURROGATE(ch) && end - s >= Py_UNICODE_SIZE)
6548	n/a	{
6549	n/a	Py_UNICODE uch2;
6550	n/a	((char *) &uch2)[0] = s[0];
6551	n/a	((char *) &uch2)[1] = s[1];
6552	n/a	if (Py_UNICODE_IS_LOW_SURROGATE(uch2))
6553	n/a	{
6554	n/a	ch = Py_UNICODE_JOIN_SURROGATES(uch, uch2);
6555	n/a	s += Py_UNICODE_SIZE;
6556	n/a	}
6557	n/a	}
6558	n/a	#endif
6559	n/a
6560	n/a	if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0)
6561	n/a	goto onError;
6562	n/a	continue;
6563	n/a
6564	n/a	error:
6565	n/a	startinpos = s - starts;
6566	n/a	if (unicode_decode_call_errorhandler_writer(
6567	n/a	errors, &errorHandler,
6568	n/a	"unicode_internal", reason,
6569	n/a	&starts, &end, &startinpos, &endinpos, &exc, &s,
6570	n/a	&writer))
6571	n/a	goto onError;
6572	n/a	}
6573	n/a
6574	n/a	Py_XDECREF(errorHandler);
6575	n/a	Py_XDECREF(exc);
6576	n/a	return _PyUnicodeWriter_Finish(&writer);
6577	n/a
6578	n/a	onError:
6579	n/a	_PyUnicodeWriter_Dealloc(&writer);
6580	n/a	Py_XDECREF(errorHandler);
6581	n/a	Py_XDECREF(exc);
6582	n/a	return NULL;
6583	n/a	}
6584	n/a
6585	n/a	/* --- Latin-1 Codec ------------------------------------------------------ */
6586	n/a
6587	n/a	PyObject *
6588	n/a	PyUnicode_DecodeLatin1(const char *s,
6589	n/a	Py_ssize_t size,
6590	n/a	const char *errors)
6591	n/a	{
6592	n/a	/* Latin-1 is equivalent to the first 256 ordinals in Unicode. */
6593	n/a	return _PyUnicode_FromUCS1((unsigned char*)s, size);
6594	n/a	}
6595	n/a
6596	n/a	/* create or adjust a UnicodeEncodeError */
6597	n/a	static void
6598	n/a	make_encode_exception(PyObject **exceptionObject,
6599	n/a	const char *encoding,
6600	n/a	PyObject *unicode,
6601	n/a	Py_ssize_t startpos, Py_ssize_t endpos,
6602	n/a	const char *reason)
6603	n/a	{
6604	n/a	if (*exceptionObject == NULL) {
6605	n/a	*exceptionObject = PyObject_CallFunction(
6606	n/a	PyExc_UnicodeEncodeError, "sOnns",
6607	n/a	encoding, unicode, startpos, endpos, reason);
6608	n/a	}
6609	n/a	else {
6610	n/a	if (PyUnicodeEncodeError_SetStart(*exceptionObject, startpos))
6611	n/a	goto onError;
6612	n/a	if (PyUnicodeEncodeError_SetEnd(*exceptionObject, endpos))
6613	n/a	goto onError;
6614	n/a	if (PyUnicodeEncodeError_SetReason(*exceptionObject, reason))
6615	n/a	goto onError;
6616	n/a	return;
6617	n/a	onError:
6618	n/a	Py_CLEAR(*exceptionObject);
6619	n/a	}
6620	n/a	}
6621	n/a
6622	n/a	/* raises a UnicodeEncodeError */
6623	n/a	static void
6624	n/a	raise_encode_exception(PyObject **exceptionObject,
6625	n/a	const char *encoding,
6626	n/a	PyObject *unicode,
6627	n/a	Py_ssize_t startpos, Py_ssize_t endpos,
6628	n/a	const char *reason)
6629	n/a	{
6630	n/a	make_encode_exception(exceptionObject,
6631	n/a	encoding, unicode, startpos, endpos, reason);
6632	n/a	if (*exceptionObject != NULL)
6633	n/a	PyCodec_StrictErrors(*exceptionObject);
6634	n/a	}
6635	n/a
6636	n/a	/* error handling callback helper:
6637	n/a	build arguments, call the callback and check the arguments,
6638	n/a	put the result into newpos and return the replacement string, which
6639	n/a	has to be freed by the caller */
6640	n/a	static PyObject *
6641	n/a	unicode_encode_call_errorhandler(const char *errors,
6642	n/a	PyObject **errorHandler,
6643	n/a	const char encoding, const char reason,
6644	n/a	PyObject unicode, PyObject *exceptionObject,
6645	n/a	Py_ssize_t startpos, Py_ssize_t endpos,
6646	n/a	Py_ssize_t *newpos)
6647	n/a	{
6648	n/a	static const char *argparse = "On;encoding error handler must return (str/bytes, int) tuple";
6649	n/a	Py_ssize_t len;
6650	n/a	PyObject *restuple;
6651	n/a	PyObject *resunicode;
6652	n/a
6653	n/a	if (*errorHandler == NULL) {
6654	n/a	*errorHandler = PyCodec_LookupError(errors);
6655	n/a	if (*errorHandler == NULL)
6656	n/a	return NULL;
6657	n/a	}
6658	n/a
6659	n/a	if (PyUnicode_READY(unicode) == -1)
6660	n/a	return NULL;
6661	n/a	len = PyUnicode_GET_LENGTH(unicode);
6662	n/a
6663	n/a	make_encode_exception(exceptionObject,
6664	n/a	encoding, unicode, startpos, endpos, reason);
6665	n/a	if (*exceptionObject == NULL)
6666	n/a	return NULL;
6667	n/a
6668	n/a	restuple = PyObject_CallFunctionObjArgs(
6669	n/a	errorHandler, exceptionObject, NULL);
6670	n/a	if (restuple == NULL)
6671	n/a	return NULL;
6672	n/a	if (!PyTuple_Check(restuple)) {
6673	n/a	PyErr_SetString(PyExc_TypeError, &argparse[3]);
6674	n/a	Py_DECREF(restuple);
6675	n/a	return NULL;
6676	n/a	}
6677	n/a	if (!PyArg_ParseTuple(restuple, argparse,
6678	n/a	&resunicode, newpos)) {
6679	n/a	Py_DECREF(restuple);
6680	n/a	return NULL;
6681	n/a	}
6682	n/a	if (!PyUnicode_Check(resunicode) && !PyBytes_Check(resunicode)) {
6683	n/a	PyErr_SetString(PyExc_TypeError, &argparse[3]);
6684	n/a	Py_DECREF(restuple);
6685	n/a	return NULL;
6686	n/a	}
6687	n/a	if (*newpos<0)
6688	n/a	newpos = len + newpos;
6689	n/a	if (newpos<0 \|\| newpos>len) {
6690	n/a	PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", *newpos);
6691	n/a	Py_DECREF(restuple);
6692	n/a	return NULL;
6693	n/a	}
6694	n/a	Py_INCREF(resunicode);
6695	n/a	Py_DECREF(restuple);
6696	n/a	return resunicode;
6697	n/a	}
6698	n/a
6699	n/a	static PyObject *
6700	n/a	unicode_encode_ucs1(PyObject *unicode,
6701	n/a	const char *errors,
6702	n/a	const Py_UCS4 limit)
6703	n/a	{
6704	n/a	/* input state */
6705	n/a	Py_ssize_t pos=0, size;
6706	n/a	int kind;
6707	n/a	void *data;
6708	n/a	/* pointer into the output */
6709	n/a	char *str;
6710	n/a	const char *encoding = (limit == 256) ? "latin-1" : "ascii";
6711	n/a	const char *reason = (limit == 256) ? "ordinal not in range(256)" : "ordinal not in range(128)";
6712	n/a	PyObject *error_handler_obj = NULL;
6713	n/a	PyObject *exc = NULL;
6714	n/a	_Py_error_handler error_handler = _Py_ERROR_UNKNOWN;
6715	n/a	PyObject *rep = NULL;
6716	n/a	/* output object */
6717	n/a	_PyBytesWriter writer;
6718	n/a
6719	n/a	if (PyUnicode_READY(unicode) == -1)
6720	n/a	return NULL;
6721	n/a	size = PyUnicode_GET_LENGTH(unicode);
6722	n/a	kind = PyUnicode_KIND(unicode);
6723	n/a	data = PyUnicode_DATA(unicode);
6724	n/a	/* allocate enough for a simple encoding without
6725	n/a	replacements, if we need more, we'll resize */
6726	n/a	if (size == 0)
6727	n/a	return PyBytes_FromStringAndSize(NULL, 0);
6728	n/a
6729	n/a	_PyBytesWriter_Init(&writer);
6730	n/a	str = _PyBytesWriter_Alloc(&writer, size);
6731	n/a	if (str == NULL)
6732	n/a	return NULL;
6733	n/a
6734	n/a	while (pos < size) {
6735	n/a	Py_UCS4 ch = PyUnicode_READ(kind, data, pos);
6736	n/a
6737	n/a	/* can we encode this? */
6738	n/a	if (ch < limit) {
6739	n/a	/* no overflow check, because we know that the space is enough */
6740	n/a	*str++ = (char)ch;
6741	n/a	++pos;
6742	n/a	}
6743	n/a	else {
6744	n/a	Py_ssize_t newpos, i;
6745	n/a	/* startpos for collecting unencodable chars */
6746	n/a	Py_ssize_t collstart = pos;
6747	n/a	Py_ssize_t collend = collstart + 1;
6748	n/a	/* find all unecodable characters */
6749	n/a
6750	n/a	while ((collend < size) && (PyUnicode_READ(kind, data, collend) >= limit))
6751	n/a	++collend;
6752	n/a
6753	n/a	/* Only overallocate the buffer if it's not the last write */
6754	n/a	writer.overallocate = (collend < size);
6755	n/a
6756	n/a	/* cache callback name lookup (if not done yet, i.e. it's the first error) */
6757	n/a	if (error_handler == _Py_ERROR_UNKNOWN)
6758	n/a	error_handler = get_error_handler(errors);
6759	n/a
6760	n/a	switch (error_handler) {
6761	n/a	case _Py_ERROR_STRICT:
6762	n/a	raise_encode_exception(&exc, encoding, unicode, collstart, collend, reason);
6763	n/a	goto onError;
6764	n/a
6765	n/a	case _Py_ERROR_REPLACE:
6766	n/a	memset(str, '?', collend - collstart);
6767	n/a	str += (collend - collstart);
6768	n/a	/* fall through ignore error handler */
6769	n/a	case _Py_ERROR_IGNORE:
6770	n/a	pos = collend;
6771	n/a	break;
6772	n/a
6773	n/a	case _Py_ERROR_BACKSLASHREPLACE:
6774	n/a	/* subtract preallocated bytes */
6775	n/a	writer.min_size -= (collend - collstart);
6776	n/a	str = backslashreplace(&writer, str,
6777	n/a	unicode, collstart, collend);
6778	n/a	if (str == NULL)
6779	n/a	goto onError;
6780	n/a	pos = collend;
6781	n/a	break;
6782	n/a
6783	n/a	case _Py_ERROR_XMLCHARREFREPLACE:
6784	n/a	/* subtract preallocated bytes */
6785	n/a	writer.min_size -= (collend - collstart);
6786	n/a	str = xmlcharrefreplace(&writer, str,
6787	n/a	unicode, collstart, collend);
6788	n/a	if (str == NULL)
6789	n/a	goto onError;
6790	n/a	pos = collend;
6791	n/a	break;
6792	n/a
6793	n/a	case _Py_ERROR_SURROGATEESCAPE:
6794	n/a	for (i = collstart; i < collend; ++i) {
6795	n/a	ch = PyUnicode_READ(kind, data, i);
6796	n/a	if (ch < 0xdc80 \|\| 0xdcff < ch) {
6797	n/a	/* Not a UTF-8b surrogate */
6798	n/a	break;
6799	n/a	}
6800	n/a	*str++ = (char)(ch - 0xdc00);
6801	n/a	++pos;
6802	n/a	}
6803	n/a	if (i >= collend)
6804	n/a	break;
6805	n/a	collstart = pos;
6806	n/a	assert(collstart != collend);
6807	n/a	/* fallback to general error handling */
6808	n/a
6809	n/a	default:
6810	n/a	rep = unicode_encode_call_errorhandler(errors, &error_handler_obj,
6811	n/a	encoding, reason, unicode, &exc,
6812	n/a	collstart, collend, &newpos);
6813	n/a	if (rep == NULL)
6814	n/a	goto onError;
6815	n/a
6816	n/a	/* subtract preallocated bytes */
6817	n/a	writer.min_size -= newpos - collstart;
6818	n/a
6819	n/a	if (PyBytes_Check(rep)) {
6820	n/a	/* Directly copy bytes result to output. */
6821	n/a	str = _PyBytesWriter_WriteBytes(&writer, str,
6822	n/a	PyBytes_AS_STRING(rep),
6823	n/a	PyBytes_GET_SIZE(rep));
6824	n/a	if (str == NULL)
6825	n/a	goto onError;
6826	n/a	}
6827	n/a	else {
6828	n/a	assert(PyUnicode_Check(rep));
6829	n/a
6830	n/a	if (PyUnicode_READY(rep) < 0)
6831	n/a	goto onError;
6832	n/a
6833	n/a	if (limit == 256 ?
6834	n/a	PyUnicode_KIND(rep) != PyUnicode_1BYTE_KIND :
6835	n/a	!PyUnicode_IS_ASCII(rep))
6836	n/a	{
6837	n/a	/* Not all characters are smaller than limit */
6838	n/a	raise_encode_exception(&exc, encoding, unicode,
6839	n/a	collstart, collend, reason);
6840	n/a	goto onError;
6841	n/a	}
6842	n/a	assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND);
6843	n/a	str = _PyBytesWriter_WriteBytes(&writer, str,
6844	n/a	PyUnicode_DATA(rep),
6845	n/a	PyUnicode_GET_LENGTH(rep));
6846	n/a	}
6847	n/a	pos = newpos;
6848	n/a	Py_CLEAR(rep);
6849	n/a	}
6850	n/a
6851	n/a	/* If overallocation was disabled, ensure that it was the last
6852	n/a	write. Otherwise, we missed an optimization */
6853	n/a	assert(writer.overallocate \|\| pos == size);
6854	n/a	}
6855	n/a	}
6856	n/a
6857	n/a	Py_XDECREF(error_handler_obj);
6858	n/a	Py_XDECREF(exc);
6859	n/a	return _PyBytesWriter_Finish(&writer, str);
6860	n/a
6861	n/a	onError:
6862	n/a	Py_XDECREF(rep);
6863	n/a	_PyBytesWriter_Dealloc(&writer);
6864	n/a	Py_XDECREF(error_handler_obj);
6865	n/a	Py_XDECREF(exc);
6866	n/a	return NULL;
6867	n/a	}
6868	n/a
6869	n/a	/* Deprecated */
6870	n/a	PyObject *
6871	n/a	PyUnicode_EncodeLatin1(const Py_UNICODE *p,
6872	n/a	Py_ssize_t size,
6873	n/a	const char *errors)
6874	n/a	{
6875	n/a	PyObject *result;
6876	n/a	PyObject *unicode = PyUnicode_FromWideChar(p, size);
6877	n/a	if (unicode == NULL)
6878	n/a	return NULL;
6879	n/a	result = unicode_encode_ucs1(unicode, errors, 256);
6880	n/a	Py_DECREF(unicode);
6881	n/a	return result;
6882	n/a	}
6883	n/a
6884	n/a	PyObject *
6885	n/a	_PyUnicode_AsLatin1String(PyObject unicode, const char errors)
6886	n/a	{
6887	n/a	if (!PyUnicode_Check(unicode)) {
6888	n/a	PyErr_BadArgument();
6889	n/a	return NULL;
6890	n/a	}
6891	n/a	if (PyUnicode_READY(unicode) == -1)
6892	n/a	return NULL;
6893	n/a	/* Fast path: if it is a one-byte string, construct
6894	n/a	bytes object directly. */
6895	n/a	if (PyUnicode_KIND(unicode) == PyUnicode_1BYTE_KIND)
6896	n/a	return PyBytes_FromStringAndSize(PyUnicode_DATA(unicode),
6897	n/a	PyUnicode_GET_LENGTH(unicode));
6898	n/a	/* Non-Latin-1 characters present. Defer to above function to
6899	n/a	raise the exception. */
6900	n/a	return unicode_encode_ucs1(unicode, errors, 256);
6901	n/a	}
6902	n/a
6903	n/a	PyObject*
6904	n/a	PyUnicode_AsLatin1String(PyObject *unicode)
6905	n/a	{
6906	n/a	return _PyUnicode_AsLatin1String(unicode, NULL);
6907	n/a	}
6908	n/a
6909	n/a	/* --- 7-bit ASCII Codec -------------------------------------------------- */
6910	n/a
6911	n/a	PyObject *
6912	n/a	PyUnicode_DecodeASCII(const char *s,
6913	n/a	Py_ssize_t size,
6914	n/a	const char *errors)
6915	n/a	{
6916	n/a	const char *starts = s;
6917	n/a	_PyUnicodeWriter writer;
6918	n/a	int kind;
6919	n/a	void *data;
6920	n/a	Py_ssize_t startinpos;
6921	n/a	Py_ssize_t endinpos;
6922	n/a	Py_ssize_t outpos;
6923	n/a	const char *e;
6924	n/a	PyObject *error_handler_obj = NULL;
6925	n/a	PyObject *exc = NULL;
6926	n/a	_Py_error_handler error_handler = _Py_ERROR_UNKNOWN;
6927	n/a
6928	n/a	if (size == 0)
6929	n/a	_Py_RETURN_UNICODE_EMPTY();
6930	n/a
6931	n/a	/* ASCII is equivalent to the first 128 ordinals in Unicode. */
6932	n/a	if (size == 1 && (unsigned char)s[0] < 128)
6933	n/a	return get_latin1_char((unsigned char)s[0]);
6934	n/a
6935	n/a	_PyUnicodeWriter_Init(&writer);
6936	n/a	writer.min_length = size;
6937	n/a	if (_PyUnicodeWriter_Prepare(&writer, writer.min_length, 127) < 0)
6938	n/a	return NULL;
6939	n/a
6940	n/a	e = s + size;
6941	n/a	data = writer.data;
6942	n/a	outpos = ascii_decode(s, e, (Py_UCS1 *)data);
6943	n/a	writer.pos = outpos;
6944	n/a	if (writer.pos == size)
6945	n/a	return _PyUnicodeWriter_Finish(&writer);
6946	n/a
6947	n/a	s += writer.pos;
6948	n/a	kind = writer.kind;
6949	n/a	while (s < e) {
6950	n/a	unsigned char c = (unsigned char)*s;
6951	n/a	if (c < 128) {
6952	n/a	PyUnicode_WRITE(kind, data, writer.pos, c);
6953	n/a	writer.pos++;
6954	n/a	++s;
6955	n/a	continue;
6956	n/a	}
6957	n/a
6958	n/a	/* byte outsize range 0x00..0x7f: call the error handler */
6959	n/a
6960	n/a	if (error_handler == _Py_ERROR_UNKNOWN)
6961	n/a	error_handler = get_error_handler(errors);
6962	n/a
6963	n/a	switch (error_handler)
6964	n/a	{
6965	n/a	case _Py_ERROR_REPLACE:
6966	n/a	case _Py_ERROR_SURROGATEESCAPE:
6967	n/a	/* Fast-path: the error handler only writes one character,
6968	n/a	but we may switch to UCS2 at the first write */
6969	n/a	if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0)
6970	n/a	goto onError;
6971	n/a	kind = writer.kind;
6972	n/a	data = writer.data;
6973	n/a
6974	n/a	if (error_handler == _Py_ERROR_REPLACE)
6975	n/a	PyUnicode_WRITE(kind, data, writer.pos, 0xfffd);
6976	n/a	else
6977	n/a	PyUnicode_WRITE(kind, data, writer.pos, c + 0xdc00);
6978	n/a	writer.pos++;
6979	n/a	++s;
6980	n/a	break;
6981	n/a
6982	n/a	case _Py_ERROR_IGNORE:
6983	n/a	++s;
6984	n/a	break;
6985	n/a
6986	n/a	default:
6987	n/a	startinpos = s-starts;
6988	n/a	endinpos = startinpos + 1;
6989	n/a	if (unicode_decode_call_errorhandler_writer(
6990	n/a	errors, &error_handler_obj,
6991	n/a	"ascii", "ordinal not in range(128)",
6992	n/a	&starts, &e, &startinpos, &endinpos, &exc, &s,
6993	n/a	&writer))
6994	n/a	goto onError;
6995	n/a	kind = writer.kind;
6996	n/a	data = writer.data;
6997	n/a	}
6998	n/a	}
6999	n/a	Py_XDECREF(error_handler_obj);
7000	n/a	Py_XDECREF(exc);
7001	n/a	return _PyUnicodeWriter_Finish(&writer);
7002	n/a
7003	n/a	onError:
7004	n/a	_PyUnicodeWriter_Dealloc(&writer);
7005	n/a	Py_XDECREF(error_handler_obj);
7006	n/a	Py_XDECREF(exc);
7007	n/a	return NULL;
7008	n/a	}
7009	n/a
7010	n/a	/* Deprecated */
7011	n/a	PyObject *
7012	n/a	PyUnicode_EncodeASCII(const Py_UNICODE *p,
7013	n/a	Py_ssize_t size,
7014	n/a	const char *errors)
7015	n/a	{
7016	n/a	PyObject *result;
7017	n/a	PyObject *unicode = PyUnicode_FromWideChar(p, size);
7018	n/a	if (unicode == NULL)
7019	n/a	return NULL;
7020	n/a	result = unicode_encode_ucs1(unicode, errors, 128);
7021	n/a	Py_DECREF(unicode);
7022	n/a	return result;
7023	n/a	}
7024	n/a
7025	n/a	PyObject *
7026	n/a	_PyUnicode_AsASCIIString(PyObject unicode, const char errors)
7027	n/a	{
7028	n/a	if (!PyUnicode_Check(unicode)) {
7029	n/a	PyErr_BadArgument();
7030	n/a	return NULL;
7031	n/a	}
7032	n/a	if (PyUnicode_READY(unicode) == -1)
7033	n/a	return NULL;
7034	n/a	/* Fast path: if it is an ASCII-only string, construct bytes object
7035	n/a	directly. Else defer to above function to raise the exception. */
7036	n/a	if (PyUnicode_IS_ASCII(unicode))
7037	n/a	return PyBytes_FromStringAndSize(PyUnicode_DATA(unicode),
7038	n/a	PyUnicode_GET_LENGTH(unicode));
7039	n/a	return unicode_encode_ucs1(unicode, errors, 128);
7040	n/a	}
7041	n/a
7042	n/a	PyObject *
7043	n/a	PyUnicode_AsASCIIString(PyObject *unicode)
7044	n/a	{
7045	n/a	return _PyUnicode_AsASCIIString(unicode, NULL);
7046	n/a	}
7047	n/a
7048	n/a	#ifdef MS_WINDOWS
7049	n/a
7050	n/a	/* --- MBCS codecs for Windows -------------------------------------------- */
7051	n/a
7052	n/a	#if SIZEOF_INT < SIZEOF_SIZE_T
7053	n/a	#define NEED_RETRY
7054	n/a	#endif
7055	n/a
7056	n/a	#ifndef WC_ERR_INVALID_CHARS
7057	n/a	# define WC_ERR_INVALID_CHARS 0x0080
7058	n/a	#endif
7059	n/a
7060	n/a	static const char*
7061	n/a	code_page_name(UINT code_page, PyObject **obj)
7062	n/a	{
7063	n/a	*obj = NULL;
7064	n/a	if (code_page == CP_ACP)
7065	n/a	return "mbcs";
7066	n/a	if (code_page == CP_UTF7)
7067	n/a	return "CP_UTF7";
7068	n/a	if (code_page == CP_UTF8)
7069	n/a	return "CP_UTF8";
7070	n/a
7071	n/a	*obj = PyBytes_FromFormat("cp%u", code_page);
7072	n/a	if (*obj == NULL)
7073	n/a	return NULL;
7074	n/a	return PyBytes_AS_STRING(*obj);
7075	n/a	}
7076	n/a
7077	n/a	static DWORD
7078	n/a	decode_code_page_flags(UINT code_page)
7079	n/a	{
7080	n/a	if (code_page == CP_UTF7) {
7081	n/a	/* The CP_UTF7 decoder only supports flags=0 */
7082	n/a	return 0;
7083	n/a	}
7084	n/a	else
7085	n/a	return MB_ERR_INVALID_CHARS;
7086	n/a	}
7087	n/a
7088	n/a	/*
7089	n/a	* Decode a byte string from a Windows code page into unicode object in strict
7090	n/a	* mode.
7091	n/a	*
7092	n/a	* Returns consumed size if succeed, returns -2 on decode error, or raise an
7093	n/a	* OSError and returns -1 on other error.
7094	n/a	*/
7095	n/a	static int
7096	n/a	decode_code_page_strict(UINT code_page,
7097	n/a	PyObject **v,
7098	n/a	const char *in,
7099	n/a	int insize)
7100	n/a	{
7101	n/a	const DWORD flags = decode_code_page_flags(code_page);
7102	n/a	wchar_t *out;
7103	n/a	DWORD outsize;
7104	n/a
7105	n/a	/* First get the size of the result */
7106	n/a	assert(insize > 0);
7107	n/a	outsize = MultiByteToWideChar(code_page, flags, in, insize, NULL, 0);
7108	n/a	if (outsize <= 0)
7109	n/a	goto error;
7110	n/a
7111	n/a	if (*v == NULL) {
7112	n/a	/* Create unicode object */
7113	n/a	/* FIXME: don't use _PyUnicode_New(), but allocate a wchar_t* buffer */
7114	n/a	v = (PyObject)_PyUnicode_New(outsize);
7115	n/a	if (*v == NULL)
7116	n/a	return -1;
7117	n/a	out = PyUnicode_AS_UNICODE(*v);
7118	n/a	}
7119	n/a	else {
7120	n/a	/* Extend unicode object */
7121	n/a	Py_ssize_t n = PyUnicode_GET_SIZE(*v);
7122	n/a	if (unicode_resize(v, n + outsize) < 0)
7123	n/a	return -1;
7124	n/a	out = PyUnicode_AS_UNICODE(*v) + n;
7125	n/a	}
7126	n/a
7127	n/a	/* Do the conversion */
7128	n/a	outsize = MultiByteToWideChar(code_page, flags, in, insize, out, outsize);
7129	n/a	if (outsize <= 0)
7130	n/a	goto error;
7131	n/a	return insize;
7132	n/a
7133	n/a	error:
7134	n/a	if (GetLastError() == ERROR_NO_UNICODE_TRANSLATION)
7135	n/a	return -2;
7136	n/a	PyErr_SetFromWindowsErr(0);
7137	n/a	return -1;
7138	n/a	}
7139	n/a
7140	n/a	/*
7141	n/a	* Decode a byte string from a code page into unicode object with an error
7142	n/a	* handler.
7143	n/a	*
7144	n/a	* Returns consumed size if succeed, or raise an OSError or
7145	n/a	* UnicodeDecodeError exception and returns -1 on error.
7146	n/a	*/
7147	n/a	static int
7148	n/a	decode_code_page_errors(UINT code_page,
7149	n/a	PyObject **v,
7150	n/a	const char *in, const int size,
7151	n/a	const char *errors, int final)
7152	n/a	{
7153	n/a	const char *startin = in;
7154	n/a	const char *endin = in + size;
7155	n/a	const DWORD flags = decode_code_page_flags(code_page);
7156	n/a	/* Ideally, we should get reason from FormatMessage. This is the Windows
7157	n/a	2000 English version of the message. */
7158	n/a	const char *reason = "No mapping for the Unicode character exists "
7159	n/a	"in the target code page.";
7160	n/a	/* each step cannot decode more than 1 character, but a character can be
7161	n/a	represented as a surrogate pair */
7162	n/a	wchar_t buffer[2], startout, out;
7163	n/a	int insize;
7164	n/a	Py_ssize_t outsize;
7165	n/a	PyObject *errorHandler = NULL;
7166	n/a	PyObject *exc = NULL;
7167	n/a	PyObject *encoding_obj = NULL;
7168	n/a	const char *encoding;
7169	n/a	DWORD err;
7170	n/a	int ret = -1;
7171	n/a
7172	n/a	assert(size > 0);
7173	n/a
7174	n/a	encoding = code_page_name(code_page, &encoding_obj);
7175	n/a	if (encoding == NULL)
7176	n/a	return -1;
7177	n/a
7178	n/a	if ((errors == NULL \|\| strcmp(errors, "strict") == 0) && final) {
7179	n/a	/* The last error was ERROR_NO_UNICODE_TRANSLATION, then we raise a
7180	n/a	UnicodeDecodeError. */
7181	n/a	make_decode_exception(&exc, encoding, in, size, 0, 0, reason);
7182	n/a	if (exc != NULL) {
7183	n/a	PyCodec_StrictErrors(exc);
7184	n/a	Py_CLEAR(exc);
7185	n/a	}
7186	n/a	goto error;
7187	n/a	}
7188	n/a
7189	n/a	if (*v == NULL) {
7190	n/a	/* Create unicode object */
7191	n/a	if (size > PY_SSIZE_T_MAX / (Py_ssize_t)Py_ARRAY_LENGTH(buffer)) {
7192	n/a	PyErr_NoMemory();
7193	n/a	goto error;
7194	n/a	}
7195	n/a	/* FIXME: don't use _PyUnicode_New(), but allocate a wchar_t* buffer */
7196	n/a	v = (PyObject)_PyUnicode_New(size * Py_ARRAY_LENGTH(buffer));
7197	n/a	if (*v == NULL)
7198	n/a	goto error;
7199	n/a	startout = PyUnicode_AS_UNICODE(*v);
7200	n/a	}
7201	n/a	else {
7202	n/a	/* Extend unicode object */
7203	n/a	Py_ssize_t n = PyUnicode_GET_SIZE(*v);
7204	n/a	if (size > (PY_SSIZE_T_MAX - n) / (Py_ssize_t)Py_ARRAY_LENGTH(buffer)) {
7205	n/a	PyErr_NoMemory();
7206	n/a	goto error;
7207	n/a	}
7208	n/a	if (unicode_resize(v, n + size * Py_ARRAY_LENGTH(buffer)) < 0)
7209	n/a	goto error;
7210	n/a	startout = PyUnicode_AS_UNICODE(*v) + n;
7211	n/a	}
7212	n/a
7213	n/a	/* Decode the byte string character per character */
7214	n/a	out = startout;
7215	n/a	while (in < endin)
7216	n/a	{
7217	n/a	/* Decode a character */
7218	n/a	insize = 1;
7219	n/a	do
7220	n/a	{
7221	n/a	outsize = MultiByteToWideChar(code_page, flags,
7222	n/a	in, insize,
7223	n/a	buffer, Py_ARRAY_LENGTH(buffer));
7224	n/a	if (outsize > 0)
7225	n/a	break;
7226	n/a	err = GetLastError();
7227	n/a	if (err != ERROR_NO_UNICODE_TRANSLATION
7228	n/a	&& err != ERROR_INSUFFICIENT_BUFFER)
7229	n/a	{
7230	n/a	PyErr_SetFromWindowsErr(0);
7231	n/a	goto error;
7232	n/a	}
7233	n/a	insize++;
7234	n/a	}
7235	n/a	/* 4=maximum length of a UTF-8 sequence */
7236	n/a	while (insize <= 4 && (in + insize) <= endin);
7237	n/a
7238	n/a	if (outsize <= 0) {
7239	n/a	Py_ssize_t startinpos, endinpos, outpos;
7240	n/a
7241	n/a	/* last character in partial decode? */
7242	n/a	if (in + insize >= endin && !final)
7243	n/a	break;
7244	n/a
7245	n/a	startinpos = in - startin;
7246	n/a	endinpos = startinpos + 1;
7247	n/a	outpos = out - PyUnicode_AS_UNICODE(*v);
7248	n/a	if (unicode_decode_call_errorhandler_wchar(
7249	n/a	errors, &errorHandler,
7250	n/a	encoding, reason,
7251	n/a	&startin, &endin, &startinpos, &endinpos, &exc, &in,
7252	n/a	v, &outpos))
7253	n/a	{
7254	n/a	goto error;
7255	n/a	}
7256	n/a	out = PyUnicode_AS_UNICODE(*v) + outpos;
7257	n/a	}
7258	n/a	else {
7259	n/a	in += insize;
7260	n/a	memcpy(out, buffer, outsize * sizeof(wchar_t));
7261	n/a	out += outsize;
7262	n/a	}
7263	n/a	}
7264	n/a
7265	n/a	/* write a NUL character at the end */
7266	n/a	*out = 0;
7267	n/a
7268	n/a	/* Extend unicode object */
7269	n/a	outsize = out - startout;
7270	n/a	assert(outsize <= PyUnicode_WSTR_LENGTH(*v));
7271	n/a	if (unicode_resize(v, outsize) < 0)
7272	n/a	goto error;
7273	n/a	/* (in - startin) <= size and size is an int */
7274	n/a	ret = Py_SAFE_DOWNCAST(in - startin, Py_ssize_t, int);
7275	n/a
7276	n/a	error:
7277	n/a	Py_XDECREF(encoding_obj);
7278	n/a	Py_XDECREF(errorHandler);
7279	n/a	Py_XDECREF(exc);
7280	n/a	return ret;
7281	n/a	}
7282	n/a
7283	n/a	static PyObject *
7284	n/a	decode_code_page_stateful(int code_page,
7285	n/a	const char *s, Py_ssize_t size,
7286	n/a	const char errors, Py_ssize_t consumed)
7287	n/a	{
7288	n/a	PyObject *v = NULL;
7289	n/a	int chunk_size, final, converted, done;
7290	n/a
7291	n/a	if (code_page < 0) {
7292	n/a	PyErr_SetString(PyExc_ValueError, "invalid code page number");
7293	n/a	return NULL;
7294	n/a	}
7295	n/a
7296	n/a	if (consumed)
7297	n/a	*consumed = 0;
7298	n/a
7299	n/a	do
7300	n/a	{
7301	n/a	#ifdef NEED_RETRY
7302	n/a	if (size > INT_MAX) {
7303	n/a	chunk_size = INT_MAX;
7304	n/a	final = 0;
7305	n/a	done = 0;
7306	n/a	}
7307	n/a	else
7308	n/a	#endif
7309	n/a	{
7310	n/a	chunk_size = (int)size;
7311	n/a	final = (consumed == NULL);
7312	n/a	done = 1;
7313	n/a	}
7314	n/a
7315	n/a	if (chunk_size == 0 && done) {
7316	n/a	if (v != NULL)
7317	n/a	break;
7318	n/a	_Py_RETURN_UNICODE_EMPTY();
7319	n/a	}
7320	n/a
7321	n/a	converted = decode_code_page_strict(code_page, &v,
7322	n/a	s, chunk_size);
7323	n/a	if (converted == -2)
7324	n/a	converted = decode_code_page_errors(code_page, &v,
7325	n/a	s, chunk_size,
7326	n/a	errors, final);
7327	n/a	assert(converted != 0 \|\| done);
7328	n/a
7329	n/a	if (converted < 0) {
7330	n/a	Py_XDECREF(v);
7331	n/a	return NULL;
7332	n/a	}
7333	n/a
7334	n/a	if (consumed)
7335	n/a	*consumed += converted;
7336	n/a
7337	n/a	s += converted;
7338	n/a	size -= converted;
7339	n/a	} while (!done);
7340	n/a
7341	n/a	return unicode_result(v);
7342	n/a	}
7343	n/a
7344	n/a	PyObject *
7345	n/a	PyUnicode_DecodeCodePageStateful(int code_page,
7346	n/a	const char *s,
7347	n/a	Py_ssize_t size,
7348	n/a	const char *errors,
7349	n/a	Py_ssize_t *consumed)
7350	n/a	{
7351	n/a	return decode_code_page_stateful(code_page, s, size, errors, consumed);
7352	n/a	}
7353	n/a
7354	n/a	PyObject *
7355	n/a	PyUnicode_DecodeMBCSStateful(const char *s,
7356	n/a	Py_ssize_t size,
7357	n/a	const char *errors,
7358	n/a	Py_ssize_t *consumed)
7359	n/a	{
7360	n/a	return decode_code_page_stateful(CP_ACP, s, size, errors, consumed);
7361	n/a	}
7362	n/a
7363	n/a	PyObject *
7364	n/a	PyUnicode_DecodeMBCS(const char *s,
7365	n/a	Py_ssize_t size,
7366	n/a	const char *errors)
7367	n/a	{
7368	n/a	return PyUnicode_DecodeMBCSStateful(s, size, errors, NULL);
7369	n/a	}
7370	n/a
7371	n/a	static DWORD
7372	n/a	encode_code_page_flags(UINT code_page, const char *errors)
7373	n/a	{
7374	n/a	if (code_page == CP_UTF8) {
7375	n/a	return WC_ERR_INVALID_CHARS;
7376	n/a	}
7377	n/a	else if (code_page == CP_UTF7) {
7378	n/a	/* CP_UTF7 only supports flags=0 */
7379	n/a	return 0;
7380	n/a	}
7381	n/a	else {
7382	n/a	if (errors != NULL && strcmp(errors, "replace") == 0)
7383	n/a	return 0;
7384	n/a	else
7385	n/a	return WC_NO_BEST_FIT_CHARS;
7386	n/a	}
7387	n/a	}
7388	n/a
7389	n/a	/*
7390	n/a	* Encode a Unicode string to a Windows code page into a byte string in strict
7391	n/a	* mode.
7392	n/a	*
7393	n/a	* Returns consumed characters if succeed, returns -2 on encode error, or raise
7394	n/a	* an OSError and returns -1 on other error.
7395	n/a	*/
7396	n/a	static int
7397	n/a	encode_code_page_strict(UINT code_page, PyObject **outbytes,
7398	n/a	PyObject *unicode, Py_ssize_t offset, int len,
7399	n/a	const char* errors)
7400	n/a	{
7401	n/a	BOOL usedDefaultChar = FALSE;
7402	n/a	BOOL *pusedDefaultChar = &usedDefaultChar;
7403	n/a	int outsize;
7404	n/a	wchar_t *p;
7405	n/a	Py_ssize_t size;
7406	n/a	const DWORD flags = encode_code_page_flags(code_page, NULL);
7407	n/a	char *out;
7408	n/a	/* Create a substring so that we can get the UTF-16 representation
7409	n/a	of just the slice under consideration. */
7410	n/a	PyObject *substring;
7411	n/a
7412	n/a	assert(len > 0);
7413	n/a
7414	n/a	if (code_page != CP_UTF8 && code_page != CP_UTF7)
7415	n/a	pusedDefaultChar = &usedDefaultChar;
7416	n/a	else
7417	n/a	pusedDefaultChar = NULL;
7418	n/a
7419	n/a	substring = PyUnicode_Substring(unicode, offset, offset+len);
7420	n/a	if (substring == NULL)
7421	n/a	return -1;
7422	n/a	p = PyUnicode_AsUnicodeAndSize(substring, &size);
7423	n/a	if (p == NULL) {
7424	n/a	Py_DECREF(substring);
7425	n/a	return -1;
7426	n/a	}
7427	n/a	assert(size <= INT_MAX);
7428	n/a
7429	n/a	/* First get the size of the result */
7430	n/a	outsize = WideCharToMultiByte(code_page, flags,
7431	n/a	p, (int)size,
7432	n/a	NULL, 0,
7433	n/a	NULL, pusedDefaultChar);
7434	n/a	if (outsize <= 0)
7435	n/a	goto error;
7436	n/a	/* If we used a default char, then we failed! */
7437	n/a	if (pusedDefaultChar && *pusedDefaultChar) {
7438	n/a	Py_DECREF(substring);
7439	n/a	return -2;
7440	n/a	}
7441	n/a
7442	n/a	if (*outbytes == NULL) {
7443	n/a	/* Create string object */
7444	n/a	*outbytes = PyBytes_FromStringAndSize(NULL, outsize);
7445	n/a	if (*outbytes == NULL) {
7446	n/a	Py_DECREF(substring);
7447	n/a	return -1;
7448	n/a	}
7449	n/a	out = PyBytes_AS_STRING(*outbytes);
7450	n/a	}
7451	n/a	else {
7452	n/a	/* Extend string object */
7453	n/a	const Py_ssize_t n = PyBytes_Size(*outbytes);
7454	n/a	if (outsize > PY_SSIZE_T_MAX - n) {
7455	n/a	PyErr_NoMemory();
7456	n/a	Py_DECREF(substring);
7457	n/a	return -1;
7458	n/a	}
7459	n/a	if (_PyBytes_Resize(outbytes, n + outsize) < 0) {
7460	n/a	Py_DECREF(substring);
7461	n/a	return -1;
7462	n/a	}
7463	n/a	out = PyBytes_AS_STRING(*outbytes) + n;
7464	n/a	}
7465	n/a
7466	n/a	/* Do the conversion */
7467	n/a	outsize = WideCharToMultiByte(code_page, flags,
7468	n/a	p, (int)size,
7469	n/a	out, outsize,
7470	n/a	NULL, pusedDefaultChar);
7471	n/a	Py_CLEAR(substring);
7472	n/a	if (outsize <= 0)
7473	n/a	goto error;
7474	n/a	if (pusedDefaultChar && *pusedDefaultChar)
7475	n/a	return -2;
7476	n/a	return 0;
7477	n/a
7478	n/a	error:
7479	n/a	Py_XDECREF(substring);
7480	n/a	if (GetLastError() == ERROR_NO_UNICODE_TRANSLATION)
7481	n/a	return -2;
7482	n/a	PyErr_SetFromWindowsErr(0);
7483	n/a	return -1;
7484	n/a	}
7485	n/a
7486	n/a	/*
7487	n/a	* Encode a Unicode string to a Windows code page into a byte string using an
7488	n/a	* error handler.
7489	n/a	*
7490	n/a	* Returns consumed characters if succeed, or raise an OSError and returns
7491	n/a	* -1 on other error.
7492	n/a	*/
7493	n/a	static int
7494	n/a	encode_code_page_errors(UINT code_page, PyObject **outbytes,
7495	n/a	PyObject *unicode, Py_ssize_t unicode_offset,
7496	n/a	Py_ssize_t insize, const char* errors)
7497	n/a	{
7498	n/a	const DWORD flags = encode_code_page_flags(code_page, errors);
7499	n/a	Py_ssize_t pos = unicode_offset;
7500	n/a	Py_ssize_t endin = unicode_offset + insize;
7501	n/a	/* Ideally, we should get reason from FormatMessage. This is the Windows
7502	n/a	2000 English version of the message. */
7503	n/a	const char *reason = "invalid character";
7504	n/a	/* 4=maximum length of a UTF-8 sequence */
7505	n/a	char buffer[4];
7506	n/a	BOOL usedDefaultChar = FALSE, *pusedDefaultChar;
7507	n/a	Py_ssize_t outsize;
7508	n/a	char *out;
7509	n/a	PyObject *errorHandler = NULL;
7510	n/a	PyObject *exc = NULL;
7511	n/a	PyObject *encoding_obj = NULL;
7512	n/a	const char *encoding;
7513	n/a	Py_ssize_t newpos, newoutsize;
7514	n/a	PyObject *rep;
7515	n/a	int ret = -1;
7516	n/a
7517	n/a	assert(insize > 0);
7518	n/a
7519	n/a	encoding = code_page_name(code_page, &encoding_obj);
7520	n/a	if (encoding == NULL)
7521	n/a	return -1;
7522	n/a
7523	n/a	if (errors == NULL \|\| strcmp(errors, "strict") == 0) {
7524	n/a	/* The last error was ERROR_NO_UNICODE_TRANSLATION,
7525	n/a	then we raise a UnicodeEncodeError. */
7526	n/a	make_encode_exception(&exc, encoding, unicode, 0, 0, reason);
7527	n/a	if (exc != NULL) {
7528	n/a	PyCodec_StrictErrors(exc);
7529	n/a	Py_DECREF(exc);
7530	n/a	}
7531	n/a	Py_XDECREF(encoding_obj);
7532	n/a	return -1;
7533	n/a	}
7534	n/a
7535	n/a	if (code_page != CP_UTF8 && code_page != CP_UTF7)
7536	n/a	pusedDefaultChar = &usedDefaultChar;
7537	n/a	else
7538	n/a	pusedDefaultChar = NULL;
7539	n/a
7540	n/a	if (Py_ARRAY_LENGTH(buffer) > PY_SSIZE_T_MAX / insize) {
7541	n/a	PyErr_NoMemory();
7542	n/a	goto error;
7543	n/a	}
7544	n/a	outsize = insize * Py_ARRAY_LENGTH(buffer);
7545	n/a
7546	n/a	if (*outbytes == NULL) {
7547	n/a	/* Create string object */
7548	n/a	*outbytes = PyBytes_FromStringAndSize(NULL, outsize);
7549	n/a	if (*outbytes == NULL)
7550	n/a	goto error;
7551	n/a	out = PyBytes_AS_STRING(*outbytes);
7552	n/a	}
7553	n/a	else {
7554	n/a	/* Extend string object */
7555	n/a	Py_ssize_t n = PyBytes_Size(*outbytes);
7556	n/a	if (n > PY_SSIZE_T_MAX - outsize) {
7557	n/a	PyErr_NoMemory();
7558	n/a	goto error;
7559	n/a	}
7560	n/a	if (_PyBytes_Resize(outbytes, n + outsize) < 0)
7561	n/a	goto error;
7562	n/a	out = PyBytes_AS_STRING(*outbytes) + n;
7563	n/a	}
7564	n/a
7565	n/a	/* Encode the string character per character */
7566	n/a	while (pos < endin)
7567	n/a	{
7568	n/a	Py_UCS4 ch = PyUnicode_READ_CHAR(unicode, pos);
7569	n/a	wchar_t chars[2];
7570	n/a	int charsize;
7571	n/a	if (ch < 0x10000) {
7572	n/a	chars[0] = (wchar_t)ch;
7573	n/a	charsize = 1;
7574	n/a	}
7575	n/a	else {
7576	n/a	chars[0] = Py_UNICODE_HIGH_SURROGATE(ch);
7577	n/a	chars[1] = Py_UNICODE_LOW_SURROGATE(ch);
7578	n/a	charsize = 2;
7579	n/a	}
7580	n/a
7581	n/a	outsize = WideCharToMultiByte(code_page, flags,
7582	n/a	chars, charsize,
7583	n/a	buffer, Py_ARRAY_LENGTH(buffer),
7584	n/a	NULL, pusedDefaultChar);
7585	n/a	if (outsize > 0) {
7586	n/a	if (pusedDefaultChar == NULL \|\| !(*pusedDefaultChar))
7587	n/a	{
7588	n/a	pos++;
7589	n/a	memcpy(out, buffer, outsize);
7590	n/a	out += outsize;
7591	n/a	continue;
7592	n/a	}
7593	n/a	}
7594	n/a	else if (GetLastError() != ERROR_NO_UNICODE_TRANSLATION) {
7595	n/a	PyErr_SetFromWindowsErr(0);
7596	n/a	goto error;
7597	n/a	}
7598	n/a
7599	n/a	rep = unicode_encode_call_errorhandler(
7600	n/a	errors, &errorHandler, encoding, reason,
7601	n/a	unicode, &exc,
7602	n/a	pos, pos + 1, &newpos);
7603	n/a	if (rep == NULL)
7604	n/a	goto error;
7605	n/a	pos = newpos;
7606	n/a
7607	n/a	if (PyBytes_Check(rep)) {
7608	n/a	outsize = PyBytes_GET_SIZE(rep);
7609	n/a	if (outsize != 1) {
7610	n/a	Py_ssize_t offset = out - PyBytes_AS_STRING(*outbytes);
7611	n/a	newoutsize = PyBytes_GET_SIZE(*outbytes) + (outsize - 1);
7612	n/a	if (_PyBytes_Resize(outbytes, newoutsize) < 0) {
7613	n/a	Py_DECREF(rep);
7614	n/a	goto error;
7615	n/a	}
7616	n/a	out = PyBytes_AS_STRING(*outbytes) + offset;
7617	n/a	}
7618	n/a	memcpy(out, PyBytes_AS_STRING(rep), outsize);
7619	n/a	out += outsize;
7620	n/a	}
7621	n/a	else {
7622	n/a	Py_ssize_t i;
7623	n/a	enum PyUnicode_Kind kind;
7624	n/a	void *data;
7625	n/a
7626	n/a	if (PyUnicode_READY(rep) == -1) {
7627	n/a	Py_DECREF(rep);
7628	n/a	goto error;
7629	n/a	}
7630	n/a
7631	n/a	outsize = PyUnicode_GET_LENGTH(rep);
7632	n/a	if (outsize != 1) {
7633	n/a	Py_ssize_t offset = out - PyBytes_AS_STRING(*outbytes);
7634	n/a	newoutsize = PyBytes_GET_SIZE(*outbytes) + (outsize - 1);
7635	n/a	if (_PyBytes_Resize(outbytes, newoutsize) < 0) {
7636	n/a	Py_DECREF(rep);
7637	n/a	goto error;
7638	n/a	}
7639	n/a	out = PyBytes_AS_STRING(*outbytes) + offset;
7640	n/a	}
7641	n/a	kind = PyUnicode_KIND(rep);
7642	n/a	data = PyUnicode_DATA(rep);
7643	n/a	for (i=0; i < outsize; i++) {
7644	n/a	Py_UCS4 ch = PyUnicode_READ(kind, data, i);
7645	n/a	if (ch > 127) {
7646	n/a	raise_encode_exception(&exc,
7647	n/a	encoding, unicode,
7648	n/a	pos, pos + 1,
7649	n/a	"unable to encode error handler result to ASCII");
7650	n/a	Py_DECREF(rep);
7651	n/a	goto error;
7652	n/a	}
7653	n/a	*out = (unsigned char)ch;
7654	n/a	out++;
7655	n/a	}
7656	n/a	}
7657	n/a	Py_DECREF(rep);
7658	n/a	}
7659	n/a	/* write a NUL byte */
7660	n/a	*out = 0;
7661	n/a	outsize = out - PyBytes_AS_STRING(*outbytes);
7662	n/a	assert(outsize <= PyBytes_GET_SIZE(*outbytes));
7663	n/a	if (_PyBytes_Resize(outbytes, outsize) < 0)
7664	n/a	goto error;
7665	n/a	ret = 0;
7666	n/a
7667	n/a	error:
7668	n/a	Py_XDECREF(encoding_obj);
7669	n/a	Py_XDECREF(errorHandler);
7670	n/a	Py_XDECREF(exc);
7671	n/a	return ret;
7672	n/a	}
7673	n/a
7674	n/a	static PyObject *
7675	n/a	encode_code_page(int code_page,
7676	n/a	PyObject *unicode,
7677	n/a	const char *errors)
7678	n/a	{
7679	n/a	Py_ssize_t len;
7680	n/a	PyObject *outbytes = NULL;
7681	n/a	Py_ssize_t offset;
7682	n/a	int chunk_len, ret, done;
7683	n/a
7684	n/a	if (!PyUnicode_Check(unicode)) {
7685	n/a	PyErr_BadArgument();
7686	n/a	return NULL;
7687	n/a	}
7688	n/a
7689	n/a	if (PyUnicode_READY(unicode) == -1)
7690	n/a	return NULL;
7691	n/a	len = PyUnicode_GET_LENGTH(unicode);
7692	n/a
7693	n/a	if (code_page < 0) {
7694	n/a	PyErr_SetString(PyExc_ValueError, "invalid code page number");
7695	n/a	return NULL;
7696	n/a	}
7697	n/a
7698	n/a	if (len == 0)
7699	n/a	return PyBytes_FromStringAndSize(NULL, 0);
7700	n/a
7701	n/a	offset = 0;
7702	n/a	do
7703	n/a	{
7704	n/a	#ifdef NEED_RETRY
7705	n/a	/* UTF-16 encoding may double the size, so use only INT_MAX/2
7706	n/a	chunks. */
7707	n/a	if (len > INT_MAX/2) {
7708	n/a	chunk_len = INT_MAX/2;
7709	n/a	done = 0;
7710	n/a	}
7711	n/a	else
7712	n/a	#endif
7713	n/a	{
7714	n/a	chunk_len = (int)len;
7715	n/a	done = 1;
7716	n/a	}
7717	n/a
7718	n/a	ret = encode_code_page_strict(code_page, &outbytes,
7719	n/a	unicode, offset, chunk_len,
7720	n/a	errors);
7721	n/a	if (ret == -2)
7722	n/a	ret = encode_code_page_errors(code_page, &outbytes,
7723	n/a	unicode, offset,
7724	n/a	chunk_len, errors);
7725	n/a	if (ret < 0) {
7726	n/a	Py_XDECREF(outbytes);
7727	n/a	return NULL;
7728	n/a	}
7729	n/a
7730	n/a	offset += chunk_len;
7731	n/a	len -= chunk_len;
7732	n/a	} while (!done);
7733	n/a
7734	n/a	return outbytes;
7735	n/a	}
7736	n/a
7737	n/a	PyObject *
7738	n/a	PyUnicode_EncodeMBCS(const Py_UNICODE *p,
7739	n/a	Py_ssize_t size,
7740	n/a	const char *errors)
7741	n/a	{
7742	n/a	PyObject unicode, res;
7743	n/a	unicode = PyUnicode_FromWideChar(p, size);
7744	n/a	if (unicode == NULL)
7745	n/a	return NULL;
7746	n/a	res = encode_code_page(CP_ACP, unicode, errors);
7747	n/a	Py_DECREF(unicode);
7748	n/a	return res;
7749	n/a	}
7750	n/a
7751	n/a	PyObject *
7752	n/a	PyUnicode_EncodeCodePage(int code_page,
7753	n/a	PyObject *unicode,
7754	n/a	const char *errors)
7755	n/a	{
7756	n/a	return encode_code_page(code_page, unicode, errors);
7757	n/a	}
7758	n/a
7759	n/a	PyObject *
7760	n/a	PyUnicode_AsMBCSString(PyObject *unicode)
7761	n/a	{
7762	n/a	return PyUnicode_EncodeCodePage(CP_ACP, unicode, NULL);
7763	n/a	}
7764	n/a
7765	n/a	#undef NEED_RETRY
7766	n/a
7767	n/a	#endif /* MS_WINDOWS */
7768	n/a
7769	n/a	/* --- Character Mapping Codec -------------------------------------------- */
7770	n/a
7771	n/a	static int
7772	n/a	charmap_decode_string(const char *s,
7773	n/a	Py_ssize_t size,
7774	n/a	PyObject *mapping,
7775	n/a	const char *errors,
7776	n/a	_PyUnicodeWriter *writer)
7777	n/a	{
7778	n/a	const char *starts = s;
7779	n/a	const char *e;
7780	n/a	Py_ssize_t startinpos, endinpos;
7781	n/a	PyObject errorHandler = NULL, exc = NULL;
7782	n/a	Py_ssize_t maplen;
7783	n/a	enum PyUnicode_Kind mapkind;
7784	n/a	void *mapdata;
7785	n/a	Py_UCS4 x;
7786	n/a	unsigned char ch;
7787	n/a
7788	n/a	if (PyUnicode_READY(mapping) == -1)
7789	n/a	return -1;
7790	n/a
7791	n/a	maplen = PyUnicode_GET_LENGTH(mapping);
7792	n/a	mapdata = PyUnicode_DATA(mapping);
7793	n/a	mapkind = PyUnicode_KIND(mapping);
7794	n/a
7795	n/a	e = s + size;
7796	n/a
7797	n/a	if (mapkind == PyUnicode_1BYTE_KIND && maplen >= 256) {
7798	n/a	/* fast-path for cp037, cp500 and iso8859_1 encodings. iso8859_1
7799	n/a	* is disabled in encoding aliases, latin1 is preferred because
7800	n/a	* its implementation is faster. */
7801	n/a	Py_UCS1 mapdata_ucs1 = (Py_UCS1 )mapdata;
7802	n/a	Py_UCS1 outdata = (Py_UCS1 )writer->data;
7803	n/a	Py_UCS4 maxchar = writer->maxchar;
7804	n/a
7805	n/a	assert (writer->kind == PyUnicode_1BYTE_KIND);
7806	n/a	while (s < e) {
7807	n/a	ch = *s;
7808	n/a	x = mapdata_ucs1[ch];
7809	n/a	if (x > maxchar) {
7810	n/a	if (_PyUnicodeWriter_Prepare(writer, 1, 0xff) == -1)
7811	n/a	goto onError;
7812	n/a	maxchar = writer->maxchar;
7813	n/a	outdata = (Py_UCS1 *)writer->data;
7814	n/a	}
7815	n/a	outdata[writer->pos] = x;
7816	n/a	writer->pos++;
7817	n/a	++s;
7818	n/a	}
7819	n/a	return 0;
7820	n/a	}
7821	n/a
7822	n/a	while (s < e) {
7823	n/a	if (mapkind == PyUnicode_2BYTE_KIND && maplen >= 256) {
7824	n/a	enum PyUnicode_Kind outkind = writer->kind;
7825	n/a	Py_UCS2 mapdata_ucs2 = (Py_UCS2 )mapdata;
7826	n/a	if (outkind == PyUnicode_1BYTE_KIND) {
7827	n/a	Py_UCS1 outdata = (Py_UCS1 )writer->data;
7828	n/a	Py_UCS4 maxchar = writer->maxchar;
7829	n/a	while (s < e) {
7830	n/a	ch = *s;
7831	n/a	x = mapdata_ucs2[ch];
7832	n/a	if (x > maxchar)
7833	n/a	goto Error;
7834	n/a	outdata[writer->pos] = x;
7835	n/a	writer->pos++;
7836	n/a	++s;
7837	n/a	}
7838	n/a	break;
7839	n/a	}
7840	n/a	else if (outkind == PyUnicode_2BYTE_KIND) {
7841	n/a	Py_UCS2 outdata = (Py_UCS2 )writer->data;
7842	n/a	while (s < e) {
7843	n/a	ch = *s;
7844	n/a	x = mapdata_ucs2[ch];
7845	n/a	if (x == 0xFFFE)
7846	n/a	goto Error;
7847	n/a	outdata[writer->pos] = x;
7848	n/a	writer->pos++;
7849	n/a	++s;
7850	n/a	}
7851	n/a	break;
7852	n/a	}
7853	n/a	}
7854	n/a	ch = *s;
7855	n/a
7856	n/a	if (ch < maplen)
7857	n/a	x = PyUnicode_READ(mapkind, mapdata, ch);
7858	n/a	else
7859	n/a	x = 0xfffe; /* invalid value */
7860	n/a	Error:
7861	n/a	if (x == 0xfffe)
7862	n/a	{
7863	n/a	/* undefined mapping */
7864	n/a	startinpos = s-starts;
7865	n/a	endinpos = startinpos+1;
7866	n/a	if (unicode_decode_call_errorhandler_writer(
7867	n/a	errors, &errorHandler,
7868	n/a	"charmap", "character maps to <undefined>",
7869	n/a	&starts, &e, &startinpos, &endinpos, &exc, &s,
7870	n/a	writer)) {
7871	n/a	goto onError;
7872	n/a	}
7873	n/a	continue;
7874	n/a	}
7875	n/a
7876	n/a	if (_PyUnicodeWriter_WriteCharInline(writer, x) < 0)
7877	n/a	goto onError;
7878	n/a	++s;
7879	n/a	}
7880	n/a	Py_XDECREF(errorHandler);
7881	n/a	Py_XDECREF(exc);
7882	n/a	return 0;
7883	n/a
7884	n/a	onError:
7885	n/a	Py_XDECREF(errorHandler);
7886	n/a	Py_XDECREF(exc);
7887	n/a	return -1;
7888	n/a	}
7889	n/a
7890	n/a	static int
7891	n/a	charmap_decode_mapping(const char *s,
7892	n/a	Py_ssize_t size,
7893	n/a	PyObject *mapping,
7894	n/a	const char *errors,
7895	n/a	_PyUnicodeWriter *writer)
7896	n/a	{
7897	n/a	const char *starts = s;
7898	n/a	const char *e;
7899	n/a	Py_ssize_t startinpos, endinpos;
7900	n/a	PyObject errorHandler = NULL, exc = NULL;
7901	n/a	unsigned char ch;
7902	n/a	PyObject key, item = NULL;
7903	n/a
7904	n/a	e = s + size;
7905	n/a
7906	n/a	while (s < e) {
7907	n/a	ch = *s;
7908	n/a
7909	n/a	/* Get mapping (char ordinal -> integer, Unicode char or None) */
7910	n/a	key = PyLong_FromLong((long)ch);
7911	n/a	if (key == NULL)
7912	n/a	goto onError;
7913	n/a
7914	n/a	item = PyObject_GetItem(mapping, key);
7915	n/a	Py_DECREF(key);
7916	n/a	if (item == NULL) {
7917	n/a	if (PyErr_ExceptionMatches(PyExc_LookupError)) {
7918	n/a	/* No mapping found means: mapping is undefined. */
7919	n/a	PyErr_Clear();
7920	n/a	goto Undefined;
7921	n/a	} else
7922	n/a	goto onError;
7923	n/a	}
7924	n/a
7925	n/a	/* Apply mapping */
7926	n/a	if (item == Py_None)
7927	n/a	goto Undefined;
7928	n/a	if (PyLong_Check(item)) {
7929	n/a	long value = PyLong_AS_LONG(item);
7930	n/a	if (value == 0xFFFE)
7931	n/a	goto Undefined;
7932	n/a	if (value < 0 \|\| value > MAX_UNICODE) {
7933	n/a	PyErr_Format(PyExc_TypeError,
7934	n/a	"character mapping must be in range(0x%lx)",
7935	n/a	(unsigned long)MAX_UNICODE + 1);
7936	n/a	goto onError;
7937	n/a	}
7938	n/a
7939	n/a	if (_PyUnicodeWriter_WriteCharInline(writer, value) < 0)
7940	n/a	goto onError;
7941	n/a	}
7942	n/a	else if (PyUnicode_Check(item)) {
7943	n/a	if (PyUnicode_READY(item) == -1)
7944	n/a	goto onError;
7945	n/a	if (PyUnicode_GET_LENGTH(item) == 1) {
7946	n/a	Py_UCS4 value = PyUnicode_READ_CHAR(item, 0);
7947	n/a	if (value == 0xFFFE)
7948	n/a	goto Undefined;
7949	n/a	if (_PyUnicodeWriter_WriteCharInline(writer, value) < 0)
7950	n/a	goto onError;
7951	n/a	}
7952	n/a	else {
7953	n/a	writer->overallocate = 1;
7954	n/a	if (_PyUnicodeWriter_WriteStr(writer, item) == -1)
7955	n/a	goto onError;
7956	n/a	}
7957	n/a	}
7958	n/a	else {
7959	n/a	/* wrong return value */
7960	n/a	PyErr_SetString(PyExc_TypeError,
7961	n/a	"character mapping must return integer, None or str");
7962	n/a	goto onError;
7963	n/a	}
7964	n/a	Py_CLEAR(item);
7965	n/a	++s;
7966	n/a	continue;
7967	n/a
7968	n/a	Undefined:
7969	n/a	/* undefined mapping */
7970	n/a	Py_CLEAR(item);
7971	n/a	startinpos = s-starts;
7972	n/a	endinpos = startinpos+1;
7973	n/a	if (unicode_decode_call_errorhandler_writer(
7974	n/a	errors, &errorHandler,
7975	n/a	"charmap", "character maps to <undefined>",
7976	n/a	&starts, &e, &startinpos, &endinpos, &exc, &s,
7977	n/a	writer)) {
7978	n/a	goto onError;
7979	n/a	}
7980	n/a	}
7981	n/a	Py_XDECREF(errorHandler);
7982	n/a	Py_XDECREF(exc);
7983	n/a	return 0;
7984	n/a
7985	n/a	onError:
7986	n/a	Py_XDECREF(item);
7987	n/a	Py_XDECREF(errorHandler);
7988	n/a	Py_XDECREF(exc);
7989	n/a	return -1;
7990	n/a	}
7991	n/a
7992	n/a	PyObject *
7993	n/a	PyUnicode_DecodeCharmap(const char *s,
7994	n/a	Py_ssize_t size,
7995	n/a	PyObject *mapping,
7996	n/a	const char *errors)
7997	n/a	{
7998	n/a	_PyUnicodeWriter writer;
7999	n/a
8000	n/a	/* Default to Latin-1 */
8001	n/a	if (mapping == NULL)
8002	n/a	return PyUnicode_DecodeLatin1(s, size, errors);
8003	n/a
8004	n/a	if (size == 0)
8005	n/a	_Py_RETURN_UNICODE_EMPTY();
8006	n/a	_PyUnicodeWriter_Init(&writer);
8007	n/a	writer.min_length = size;
8008	n/a	if (_PyUnicodeWriter_Prepare(&writer, writer.min_length, 127) == -1)
8009	n/a	goto onError;
8010	n/a
8011	n/a	if (PyUnicode_CheckExact(mapping)) {
8012	n/a	if (charmap_decode_string(s, size, mapping, errors, &writer) < 0)
8013	n/a	goto onError;
8014	n/a	}
8015	n/a	else {
8016	n/a	if (charmap_decode_mapping(s, size, mapping, errors, &writer) < 0)
8017	n/a	goto onError;
8018	n/a	}
8019	n/a	return _PyUnicodeWriter_Finish(&writer);
8020	n/a
8021	n/a	onError:
8022	n/a	_PyUnicodeWriter_Dealloc(&writer);
8023	n/a	return NULL;
8024	n/a	}
8025	n/a
8026	n/a	/* Charmap encoding: the lookup table */
8027	n/a
8028	n/a	struct encoding_map {
8029	n/a	PyObject_HEAD
8030	n/a	unsigned char level1[32];
8031	n/a	int count2, count3;
8032	n/a	unsigned char level23[1];
8033	n/a	};
8034	n/a
8035	n/a	static PyObject*
8036	n/a	encoding_map_size(PyObject obj, PyObject args)
8037	n/a	{
8038	n/a	struct encoding_map map = (struct encoding_map)obj;
8039	n/a	return PyLong_FromLong(sizeof(map) - 1 + 16map->count2 +
8040	n/a	128*map->count3);
8041	n/a	}
8042	n/a
8043	n/a	static PyMethodDef encoding_map_methods[] = {
8044	n/a	{"size", encoding_map_size, METH_NOARGS,
8045	n/a	PyDoc_STR("Return the size (in bytes) of this object") },
8046	n/a	{ 0 }
8047	n/a	};
8048	n/a
8049	n/a	static void
8050	n/a	encoding_map_dealloc(PyObject* o)
8051	n/a	{
8052	n/a	PyObject_FREE(o);
8053	n/a	}
8054	n/a
8055	n/a	static PyTypeObject EncodingMapType = {
8056	n/a	PyVarObject_HEAD_INIT(NULL, 0)
8057	n/a	"EncodingMap", /tp_name/
8058	n/a	sizeof(struct encoding_map), /tp_basicsize/
8059	n/a	0, /tp_itemsize/
8060	n/a	/* methods */
8061	n/a	encoding_map_dealloc, /tp_dealloc/
8062	n/a	0, /tp_print/
8063	n/a	0, /tp_getattr/
8064	n/a	0, /tp_setattr/
8065	n/a	0, /tp_reserved/
8066	n/a	0, /tp_repr/
8067	n/a	0, /tp_as_number/
8068	n/a	0, /tp_as_sequence/
8069	n/a	0, /tp_as_mapping/
8070	n/a	0, /tp_hash/
8071	n/a	0, /tp_call/
8072	n/a	0, /tp_str/
8073	n/a	0, /tp_getattro/
8074	n/a	0, /tp_setattro/
8075	n/a	0, /tp_as_buffer/
8076	n/a	Py_TPFLAGS_DEFAULT, /tp_flags/
8077	n/a	0, /tp_doc/
8078	n/a	0, /tp_traverse/
8079	n/a	0, /tp_clear/
8080	n/a	0, /tp_richcompare/
8081	n/a	0, /tp_weaklistoffset/
8082	n/a	0, /tp_iter/
8083	n/a	0, /tp_iternext/
8084	n/a	encoding_map_methods, /tp_methods/
8085	n/a	0, /tp_members/
8086	n/a	0, /tp_getset/
8087	n/a	0, /tp_base/
8088	n/a	0, /tp_dict/
8089	n/a	0, /tp_descr_get/
8090	n/a	0, /tp_descr_set/
8091	n/a	0, /tp_dictoffset/
8092	n/a	0, /tp_init/
8093	n/a	0, /tp_alloc/
8094	n/a	0, /tp_new/
8095	n/a	0, /tp_free/
8096	n/a	0, /tp_is_gc/
8097	n/a	};
8098	n/a
8099	n/a	PyObject*
8100	n/a	PyUnicode_BuildEncodingMap(PyObject* string)
8101	n/a	{
8102	n/a	PyObject *result;
8103	n/a	struct encoding_map *mresult;
8104	n/a	int i;
8105	n/a	int need_dict = 0;
8106	n/a	unsigned char level1[32];
8107	n/a	unsigned char level2[512];
8108	n/a	unsigned char mlevel1, mlevel2, *mlevel3;
8109	n/a	int count2 = 0, count3 = 0;
8110	n/a	int kind;
8111	n/a	void *data;
8112	n/a	Py_ssize_t length;
8113	n/a	Py_UCS4 ch;
8114	n/a
8115	n/a	if (!PyUnicode_Check(string) \|\| !PyUnicode_GET_LENGTH(string)) {
8116	n/a	PyErr_BadArgument();
8117	n/a	return NULL;
8118	n/a	}
8119	n/a	kind = PyUnicode_KIND(string);
8120	n/a	data = PyUnicode_DATA(string);
8121	n/a	length = PyUnicode_GET_LENGTH(string);
8122	n/a	length = Py_MIN(length, 256);
8123	n/a	memset(level1, 0xFF, sizeof level1);
8124	n/a	memset(level2, 0xFF, sizeof level2);
8125	n/a
8126	n/a	/* If there isn't a one-to-one mapping of NULL to \0,
8127	n/a	or if there are non-BMP characters, we need to use
8128	n/a	a mapping dictionary. */
8129	n/a	if (PyUnicode_READ(kind, data, 0) != 0)
8130	n/a	need_dict = 1;
8131	n/a	for (i = 1; i < length; i++) {
8132	n/a	int l1, l2;
8133	n/a	ch = PyUnicode_READ(kind, data, i);
8134	n/a	if (ch == 0 \|\| ch > 0xFFFF) {
8135	n/a	need_dict = 1;
8136	n/a	break;
8137	n/a	}
8138	n/a	if (ch == 0xFFFE)
8139	n/a	/* unmapped character */
8140	n/a	continue;
8141	n/a	l1 = ch >> 11;
8142	n/a	l2 = ch >> 7;
8143	n/a	if (level1[l1] == 0xFF)
8144	n/a	level1[l1] = count2++;
8145	n/a	if (level2[l2] == 0xFF)
8146	n/a	level2[l2] = count3++;
8147	n/a	}
8148	n/a
8149	n/a	if (count2 >= 0xFF \|\| count3 >= 0xFF)
8150	n/a	need_dict = 1;
8151	n/a
8152	n/a	if (need_dict) {
8153	n/a	PyObject *result = PyDict_New();
8154	n/a	PyObject key, value;
8155	n/a	if (!result)
8156	n/a	return NULL;
8157	n/a	for (i = 0; i < length; i++) {
8158	n/a	key = PyLong_FromLong(PyUnicode_READ(kind, data, i));
8159	n/a	value = PyLong_FromLong(i);
8160	n/a	if (!key \|\| !value)
8161	n/a	goto failed1;
8162	n/a	if (PyDict_SetItem(result, key, value) == -1)
8163	n/a	goto failed1;
8164	n/a	Py_DECREF(key);
8165	n/a	Py_DECREF(value);
8166	n/a	}
8167	n/a	return result;
8168	n/a	failed1:
8169	n/a	Py_XDECREF(key);
8170	n/a	Py_XDECREF(value);
8171	n/a	Py_DECREF(result);
8172	n/a	return NULL;
8173	n/a	}
8174	n/a
8175	n/a	/* Create a three-level trie */
8176	n/a	result = PyObject_MALLOC(sizeof(struct encoding_map) +
8177	n/a	16count2 + 128count3 - 1);
8178	n/a	if (!result)
8179	n/a	return PyErr_NoMemory();
8180	n/a	PyObject_Init(result, &EncodingMapType);
8181	n/a	mresult = (struct encoding_map*)result;
8182	n/a	mresult->count2 = count2;
8183	n/a	mresult->count3 = count3;
8184	n/a	mlevel1 = mresult->level1;
8185	n/a	mlevel2 = mresult->level23;
8186	n/a	mlevel3 = mresult->level23 + 16*count2;
8187	n/a	memcpy(mlevel1, level1, 32);
8188	n/a	memset(mlevel2, 0xFF, 16*count2);
8189	n/a	memset(mlevel3, 0, 128*count3);
8190	n/a	count3 = 0;
8191	n/a	for (i = 1; i < length; i++) {
8192	n/a	int o1, o2, o3, i2, i3;
8193	n/a	Py_UCS4 ch = PyUnicode_READ(kind, data, i);
8194	n/a	if (ch == 0xFFFE)
8195	n/a	/* unmapped character */
8196	n/a	continue;
8197	n/a	o1 = ch>>11;
8198	n/a	o2 = (ch>>7) & 0xF;
8199	n/a	i2 = 16*mlevel1[o1] + o2;
8200	n/a	if (mlevel2[i2] == 0xFF)
8201	n/a	mlevel2[i2] = count3++;
8202	n/a	o3 = ch & 0x7F;
8203	n/a	i3 = 128*mlevel2[i2] + o3;
8204	n/a	mlevel3[i3] = i;
8205	n/a	}
8206	n/a	return result;
8207	n/a	}
8208	n/a
8209	n/a	static int
8210	n/a	encoding_map_lookup(Py_UCS4 c, PyObject *mapping)
8211	n/a	{
8212	n/a	struct encoding_map map = (struct encoding_map)mapping;
8213	n/a	int l1 = c>>11;
8214	n/a	int l2 = (c>>7) & 0xF;
8215	n/a	int l3 = c & 0x7F;
8216	n/a	int i;
8217	n/a
8218	n/a	if (c > 0xFFFF)
8219	n/a	return -1;
8220	n/a	if (c == 0)
8221	n/a	return 0;
8222	n/a	/* level 1*/
8223	n/a	i = map->level1[l1];
8224	n/a	if (i == 0xFF) {
8225	n/a	return -1;
8226	n/a	}
8227	n/a	/* level 2*/
8228	n/a	i = map->level23[16*i+l2];
8229	n/a	if (i == 0xFF) {
8230	n/a	return -1;
8231	n/a	}
8232	n/a	/* level 3 */
8233	n/a	i = map->level23[16map->count2 + 128i + l3];
8234	n/a	if (i == 0) {
8235	n/a	return -1;
8236	n/a	}
8237	n/a	return i;
8238	n/a	}
8239	n/a
8240	n/a	/* Lookup the character ch in the mapping. If the character
8241	n/a	can't be found, Py_None is returned (or NULL, if another
8242	n/a	error occurred). */
8243	n/a	static PyObject *
8244	n/a	charmapencode_lookup(Py_UCS4 c, PyObject *mapping)
8245	n/a	{
8246	n/a	PyObject *w = PyLong_FromLong((long)c);
8247	n/a	PyObject *x;
8248	n/a
8249	n/a	if (w == NULL)
8250	n/a	return NULL;
8251	n/a	x = PyObject_GetItem(mapping, w);
8252	n/a	Py_DECREF(w);
8253	n/a	if (x == NULL) {
8254	n/a	if (PyErr_ExceptionMatches(PyExc_LookupError)) {
8255	n/a	/* No mapping found means: mapping is undefined. */
8256	n/a	PyErr_Clear();
8257	n/a	Py_RETURN_NONE;
8258	n/a	} else
8259	n/a	return NULL;
8260	n/a	}
8261	n/a	else if (x == Py_None)
8262	n/a	return x;
8263	n/a	else if (PyLong_Check(x)) {
8264	n/a	long value = PyLong_AS_LONG(x);
8265	n/a	if (value < 0 \|\| value > 255) {
8266	n/a	PyErr_SetString(PyExc_TypeError,
8267	n/a	"character mapping must be in range(256)");
8268	n/a	Py_DECREF(x);
8269	n/a	return NULL;
8270	n/a	}
8271	n/a	return x;
8272	n/a	}
8273	n/a	else if (PyBytes_Check(x))
8274	n/a	return x;
8275	n/a	else {
8276	n/a	/* wrong return value */
8277	n/a	PyErr_Format(PyExc_TypeError,
8278	n/a	"character mapping must return integer, bytes or None, not %.400s",
8279	n/a	x->ob_type->tp_name);
8280	n/a	Py_DECREF(x);
8281	n/a	return NULL;
8282	n/a	}
8283	n/a	}
8284	n/a
8285	n/a	static int
8286	n/a	charmapencode_resize(PyObject *outobj, Py_ssize_t outpos, Py_ssize_t requiredsize)
8287	n/a	{
8288	n/a	Py_ssize_t outsize = PyBytes_GET_SIZE(*outobj);
8289	n/a	/* exponentially overallocate to minimize reallocations */
8290	n/a	if (requiredsize < 2*outsize)
8291	n/a	requiredsize = 2*outsize;
8292	n/a	if (_PyBytes_Resize(outobj, requiredsize))
8293	n/a	return -1;
8294	n/a	return 0;
8295	n/a	}
8296	n/a
8297	n/a	typedef enum charmapencode_result {
8298	n/a	enc_SUCCESS, enc_FAILED, enc_EXCEPTION
8299	n/a	} charmapencode_result;
8300	n/a	/* lookup the character, put the result in the output string and adjust
8301	n/a	various state variables. Resize the output bytes object if not enough
8302	n/a	space is available. Return a new reference to the object that
8303	n/a	was put in the output buffer, or Py_None, if the mapping was undefined
8304	n/a	(in which case no character was written) or NULL, if a
8305	n/a	reallocation error occurred. The caller must decref the result */
8306	n/a	static charmapencode_result
8307	n/a	charmapencode_output(Py_UCS4 c, PyObject *mapping,
8308	n/a	PyObject *outobj, Py_ssize_t outpos)
8309	n/a	{
8310	n/a	PyObject *rep;
8311	n/a	char *outstart;
8312	n/a	Py_ssize_t outsize = PyBytes_GET_SIZE(*outobj);
8313	n/a
8314	n/a	if (Py_TYPE(mapping) == &EncodingMapType) {
8315	n/a	int res = encoding_map_lookup(c, mapping);
8316	n/a	Py_ssize_t requiredsize = *outpos+1;
8317	n/a	if (res == -1)
8318	n/a	return enc_FAILED;
8319	n/a	if (outsize<requiredsize)
8320	n/a	if (charmapencode_resize(outobj, outpos, requiredsize))
8321	n/a	return enc_EXCEPTION;
8322	n/a	outstart = PyBytes_AS_STRING(*outobj);
8323	n/a	outstart[(*outpos)++] = (char)res;
8324	n/a	return enc_SUCCESS;
8325	n/a	}
8326	n/a
8327	n/a	rep = charmapencode_lookup(c, mapping);
8328	n/a	if (rep==NULL)
8329	n/a	return enc_EXCEPTION;
8330	n/a	else if (rep==Py_None) {
8331	n/a	Py_DECREF(rep);
8332	n/a	return enc_FAILED;
8333	n/a	} else {
8334	n/a	if (PyLong_Check(rep)) {
8335	n/a	Py_ssize_t requiredsize = *outpos+1;
8336	n/a	if (outsize<requiredsize)
8337	n/a	if (charmapencode_resize(outobj, outpos, requiredsize)) {
8338	n/a	Py_DECREF(rep);
8339	n/a	return enc_EXCEPTION;
8340	n/a	}
8341	n/a	outstart = PyBytes_AS_STRING(*outobj);
8342	n/a	outstart[(*outpos)++] = (char)PyLong_AS_LONG(rep);
8343	n/a	}
8344	n/a	else {
8345	n/a	const char *repchars = PyBytes_AS_STRING(rep);
8346	n/a	Py_ssize_t repsize = PyBytes_GET_SIZE(rep);
8347	n/a	Py_ssize_t requiredsize = *outpos+repsize;
8348	n/a	if (outsize<requiredsize)
8349	n/a	if (charmapencode_resize(outobj, outpos, requiredsize)) {
8350	n/a	Py_DECREF(rep);
8351	n/a	return enc_EXCEPTION;
8352	n/a	}
8353	n/a	outstart = PyBytes_AS_STRING(*outobj);
8354	n/a	memcpy(outstart + *outpos, repchars, repsize);
8355	n/a	*outpos += repsize;
8356	n/a	}
8357	n/a	}
8358	n/a	Py_DECREF(rep);
8359	n/a	return enc_SUCCESS;
8360	n/a	}
8361	n/a
8362	n/a	/* handle an error in PyUnicode_EncodeCharmap
8363	n/a	Return 0 on success, -1 on error */
8364	n/a	static int
8365	n/a	charmap_encoding_error(
8366	n/a	PyObject unicode, Py_ssize_t inpos, PyObject *mapping,
8367	n/a	PyObject **exceptionObject,
8368	n/a	_Py_error_handler error_handler, PyObject error_handler_obj, const char errors,
8369	n/a	PyObject *res, Py_ssize_t respos)
8370	n/a	{
8371	n/a	PyObject repunicode = NULL; / initialize to prevent gcc warning */
8372	n/a	Py_ssize_t size, repsize;
8373	n/a	Py_ssize_t newpos;
8374	n/a	enum PyUnicode_Kind kind;
8375	n/a	void *data;
8376	n/a	Py_ssize_t index;
8377	n/a	/* startpos for collecting unencodable chars */
8378	n/a	Py_ssize_t collstartpos = *inpos;
8379	n/a	Py_ssize_t collendpos = *inpos+1;
8380	n/a	Py_ssize_t collpos;
8381	n/a	char *encoding = "charmap";
8382	n/a	char *reason = "character maps to <undefined>";
8383	n/a	charmapencode_result x;
8384	n/a	Py_UCS4 ch;
8385	n/a	int val;
8386	n/a
8387	n/a	if (PyUnicode_READY(unicode) == -1)
8388	n/a	return -1;
8389	n/a	size = PyUnicode_GET_LENGTH(unicode);
8390	n/a	/* find all unencodable characters */
8391	n/a	while (collendpos < size) {
8392	n/a	PyObject *rep;
8393	n/a	if (Py_TYPE(mapping) == &EncodingMapType) {
8394	n/a	ch = PyUnicode_READ_CHAR(unicode, collendpos);
8395	n/a	val = encoding_map_lookup(ch, mapping);
8396	n/a	if (val != -1)
8397	n/a	break;
8398	n/a	++collendpos;
8399	n/a	continue;
8400	n/a	}
8401	n/a
8402	n/a	ch = PyUnicode_READ_CHAR(unicode, collendpos);
8403	n/a	rep = charmapencode_lookup(ch, mapping);
8404	n/a	if (rep==NULL)
8405	n/a	return -1;
8406	n/a	else if (rep!=Py_None) {
8407	n/a	Py_DECREF(rep);
8408	n/a	break;
8409	n/a	}
8410	n/a	Py_DECREF(rep);
8411	n/a	++collendpos;
8412	n/a	}
8413	n/a	/* cache callback name lookup
8414	n/a	* (if not done yet, i.e. it's the first error) */
8415	n/a	if (*error_handler == _Py_ERROR_UNKNOWN)
8416	n/a	*error_handler = get_error_handler(errors);
8417	n/a
8418	n/a	switch (*error_handler) {
8419	n/a	case _Py_ERROR_STRICT:
8420	n/a	raise_encode_exception(exceptionObject, encoding, unicode, collstartpos, collendpos, reason);
8421	n/a	return -1;
8422	n/a
8423	n/a	case _Py_ERROR_REPLACE:
8424	n/a	for (collpos = collstartpos; collpos<collendpos; ++collpos) {
8425	n/a	x = charmapencode_output('?', mapping, res, respos);
8426	n/a	if (x==enc_EXCEPTION) {
8427	n/a	return -1;
8428	n/a	}
8429	n/a	else if (x==enc_FAILED) {
8430	n/a	raise_encode_exception(exceptionObject, encoding, unicode, collstartpos, collendpos, reason);
8431	n/a	return -1;
8432	n/a	}
8433	n/a	}
8434	n/a	/* fall through */
8435	n/a	case _Py_ERROR_IGNORE:
8436	n/a	*inpos = collendpos;
8437	n/a	break;
8438	n/a
8439	n/a	case _Py_ERROR_XMLCHARREFREPLACE:
8440	n/a	/* generate replacement (temporarily (mis)uses p) */
8441	n/a	for (collpos = collstartpos; collpos < collendpos; ++collpos) {
8442	n/a	char buffer[2+29+1+1];
8443	n/a	char *cp;
8444	n/a	sprintf(buffer, "&#%d;", (int)PyUnicode_READ_CHAR(unicode, collpos));
8445	n/a	for (cp = buffer; *cp; ++cp) {
8446	n/a	x = charmapencode_output(*cp, mapping, res, respos);
8447	n/a	if (x==enc_EXCEPTION)
8448	n/a	return -1;
8449	n/a	else if (x==enc_FAILED) {
8450	n/a	raise_encode_exception(exceptionObject, encoding, unicode, collstartpos, collendpos, reason);
8451	n/a	return -1;
8452	n/a	}
8453	n/a	}
8454	n/a	}
8455	n/a	*inpos = collendpos;
8456	n/a	break;
8457	n/a
8458	n/a	default:
8459	n/a	repunicode = unicode_encode_call_errorhandler(errors, error_handler_obj,
8460	n/a	encoding, reason, unicode, exceptionObject,
8461	n/a	collstartpos, collendpos, &newpos);
8462	n/a	if (repunicode == NULL)
8463	n/a	return -1;
8464	n/a	if (PyBytes_Check(repunicode)) {
8465	n/a	/* Directly copy bytes result to output. */
8466	n/a	Py_ssize_t outsize = PyBytes_Size(*res);
8467	n/a	Py_ssize_t requiredsize;
8468	n/a	repsize = PyBytes_Size(repunicode);
8469	n/a	requiredsize = *respos + repsize;
8470	n/a	if (requiredsize > outsize)
8471	n/a	/* Make room for all additional bytes. */
8472	n/a	if (charmapencode_resize(res, respos, requiredsize)) {
8473	n/a	Py_DECREF(repunicode);
8474	n/a	return -1;
8475	n/a	}
8476	n/a	memcpy(PyBytes_AsString(res) + respos,
8477	n/a	PyBytes_AsString(repunicode), repsize);
8478	n/a	*respos += repsize;
8479	n/a	*inpos = newpos;
8480	n/a	Py_DECREF(repunicode);
8481	n/a	break;
8482	n/a	}
8483	n/a	/* generate replacement */
8484	n/a	if (PyUnicode_READY(repunicode) == -1) {
8485	n/a	Py_DECREF(repunicode);
8486	n/a	return -1;
8487	n/a	}
8488	n/a	repsize = PyUnicode_GET_LENGTH(repunicode);
8489	n/a	data = PyUnicode_DATA(repunicode);
8490	n/a	kind = PyUnicode_KIND(repunicode);
8491	n/a	for (index = 0; index < repsize; index++) {
8492	n/a	Py_UCS4 repch = PyUnicode_READ(kind, data, index);
8493	n/a	x = charmapencode_output(repch, mapping, res, respos);
8494	n/a	if (x==enc_EXCEPTION) {
8495	n/a	Py_DECREF(repunicode);
8496	n/a	return -1;
8497	n/a	}
8498	n/a	else if (x==enc_FAILED) {
8499	n/a	Py_DECREF(repunicode);
8500	n/a	raise_encode_exception(exceptionObject, encoding, unicode, collstartpos, collendpos, reason);
8501	n/a	return -1;
8502	n/a	}
8503	n/a	}
8504	n/a	*inpos = newpos;
8505	n/a	Py_DECREF(repunicode);
8506	n/a	}
8507	n/a	return 0;
8508	n/a	}
8509	n/a
8510	n/a	PyObject *
8511	n/a	_PyUnicode_EncodeCharmap(PyObject *unicode,
8512	n/a	PyObject *mapping,
8513	n/a	const char *errors)
8514	n/a	{
8515	n/a	/* output object */
8516	n/a	PyObject *res = NULL;
8517	n/a	/* current input position */
8518	n/a	Py_ssize_t inpos = 0;
8519	n/a	Py_ssize_t size;
8520	n/a	/* current output position */
8521	n/a	Py_ssize_t respos = 0;
8522	n/a	PyObject *error_handler_obj = NULL;
8523	n/a	PyObject *exc = NULL;
8524	n/a	_Py_error_handler error_handler = _Py_ERROR_UNKNOWN;
8525	n/a	void *data;
8526	n/a	int kind;
8527	n/a
8528	n/a	if (PyUnicode_READY(unicode) == -1)
8529	n/a	return NULL;
8530	n/a	size = PyUnicode_GET_LENGTH(unicode);
8531	n/a	data = PyUnicode_DATA(unicode);
8532	n/a	kind = PyUnicode_KIND(unicode);
8533	n/a
8534	n/a	/* Default to Latin-1 */
8535	n/a	if (mapping == NULL)
8536	n/a	return unicode_encode_ucs1(unicode, errors, 256);
8537	n/a
8538	n/a	/* allocate enough for a simple encoding without
8539	n/a	replacements, if we need more, we'll resize */
8540	n/a	res = PyBytes_FromStringAndSize(NULL, size);
8541	n/a	if (res == NULL)
8542	n/a	goto onError;
8543	n/a	if (size == 0)
8544	n/a	return res;
8545	n/a
8546	n/a	while (inpos<size) {
8547	n/a	Py_UCS4 ch = PyUnicode_READ(kind, data, inpos);
8548	n/a	/* try to encode it */
8549	n/a	charmapencode_result x = charmapencode_output(ch, mapping, &res, &respos);
8550	n/a	if (x==enc_EXCEPTION) /* error */
8551	n/a	goto onError;
8552	n/a	if (x==enc_FAILED) { /* unencodable character */
8553	n/a	if (charmap_encoding_error(unicode, &inpos, mapping,
8554	n/a	&exc,
8555	n/a	&error_handler, &error_handler_obj, errors,
8556	n/a	&res, &respos)) {
8557	n/a	goto onError;
8558	n/a	}
8559	n/a	}
8560	n/a	else
8561	n/a	/* done with this character => adjust input position */
8562	n/a	++inpos;
8563	n/a	}
8564	n/a
8565	n/a	/* Resize if we allocated to much */
8566	n/a	if (respos<PyBytes_GET_SIZE(res))
8567	n/a	if (_PyBytes_Resize(&res, respos) < 0)
8568	n/a	goto onError;
8569	n/a
8570	n/a	Py_XDECREF(exc);
8571	n/a	Py_XDECREF(error_handler_obj);
8572	n/a	return res;
8573	n/a
8574	n/a	onError:
8575	n/a	Py_XDECREF(res);
8576	n/a	Py_XDECREF(exc);
8577	n/a	Py_XDECREF(error_handler_obj);
8578	n/a	return NULL;
8579	n/a	}
8580	n/a
8581	n/a	/* Deprecated */
8582	n/a	PyObject *
8583	n/a	PyUnicode_EncodeCharmap(const Py_UNICODE *p,
8584	n/a	Py_ssize_t size,
8585	n/a	PyObject *mapping,
8586	n/a	const char *errors)
8587	n/a	{
8588	n/a	PyObject *result;
8589	n/a	PyObject *unicode = PyUnicode_FromWideChar(p, size);
8590	n/a	if (unicode == NULL)
8591	n/a	return NULL;
8592	n/a	result = _PyUnicode_EncodeCharmap(unicode, mapping, errors);
8593	n/a	Py_DECREF(unicode);
8594	n/a	return result;
8595	n/a	}
8596	n/a
8597	n/a	PyObject *
8598	n/a	PyUnicode_AsCharmapString(PyObject *unicode,
8599	n/a	PyObject *mapping)
8600	n/a	{
8601	n/a	if (!PyUnicode_Check(unicode) \|\| mapping == NULL) {
8602	n/a	PyErr_BadArgument();
8603	n/a	return NULL;
8604	n/a	}
8605	n/a	return _PyUnicode_EncodeCharmap(unicode, mapping, NULL);
8606	n/a	}
8607	n/a
8608	n/a	/* create or adjust a UnicodeTranslateError */
8609	n/a	static void
8610	n/a	make_translate_exception(PyObject **exceptionObject,
8611	n/a	PyObject *unicode,
8612	n/a	Py_ssize_t startpos, Py_ssize_t endpos,
8613	n/a	const char *reason)
8614	n/a	{
8615	n/a	if (*exceptionObject == NULL) {
8616	n/a	*exceptionObject = _PyUnicodeTranslateError_Create(
8617	n/a	unicode, startpos, endpos, reason);
8618	n/a	}
8619	n/a	else {
8620	n/a	if (PyUnicodeTranslateError_SetStart(*exceptionObject, startpos))
8621	n/a	goto onError;
8622	n/a	if (PyUnicodeTranslateError_SetEnd(*exceptionObject, endpos))
8623	n/a	goto onError;
8624	n/a	if (PyUnicodeTranslateError_SetReason(*exceptionObject, reason))
8625	n/a	goto onError;
8626	n/a	return;
8627	n/a	onError:
8628	n/a	Py_CLEAR(*exceptionObject);
8629	n/a	}
8630	n/a	}
8631	n/a
8632	n/a	/* error handling callback helper:
8633	n/a	build arguments, call the callback and check the arguments,
8634	n/a	put the result into newpos and return the replacement string, which
8635	n/a	has to be freed by the caller */
8636	n/a	static PyObject *
8637	n/a	unicode_translate_call_errorhandler(const char *errors,
8638	n/a	PyObject **errorHandler,
8639	n/a	const char *reason,
8640	n/a	PyObject unicode, PyObject *exceptionObject,
8641	n/a	Py_ssize_t startpos, Py_ssize_t endpos,
8642	n/a	Py_ssize_t *newpos)
8643	n/a	{
8644	n/a	static const char *argparse = "Un;translating error handler must return (str, int) tuple";
8645	n/a
8646	n/a	Py_ssize_t i_newpos;
8647	n/a	PyObject *restuple;
8648	n/a	PyObject *resunicode;
8649	n/a
8650	n/a	if (*errorHandler == NULL) {
8651	n/a	*errorHandler = PyCodec_LookupError(errors);
8652	n/a	if (*errorHandler == NULL)
8653	n/a	return NULL;
8654	n/a	}
8655	n/a
8656	n/a	make_translate_exception(exceptionObject,
8657	n/a	unicode, startpos, endpos, reason);
8658	n/a	if (*exceptionObject == NULL)
8659	n/a	return NULL;
8660	n/a
8661	n/a	restuple = PyObject_CallFunctionObjArgs(
8662	n/a	errorHandler, exceptionObject, NULL);
8663	n/a	if (restuple == NULL)
8664	n/a	return NULL;
8665	n/a	if (!PyTuple_Check(restuple)) {
8666	n/a	PyErr_SetString(PyExc_TypeError, &argparse[3]);
8667	n/a	Py_DECREF(restuple);
8668	n/a	return NULL;
8669	n/a	}
8670	n/a	if (!PyArg_ParseTuple(restuple, argparse,
8671	n/a	&resunicode, &i_newpos)) {
8672	n/a	Py_DECREF(restuple);
8673	n/a	return NULL;
8674	n/a	}
8675	n/a	if (i_newpos<0)
8676	n/a	*newpos = PyUnicode_GET_LENGTH(unicode)+i_newpos;
8677	n/a	else
8678	n/a	*newpos = i_newpos;
8679	n/a	if (newpos<0 \|\| newpos>PyUnicode_GET_LENGTH(unicode)) {
8680	n/a	PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", *newpos);
8681	n/a	Py_DECREF(restuple);
8682	n/a	return NULL;
8683	n/a	}
8684	n/a	Py_INCREF(resunicode);
8685	n/a	Py_DECREF(restuple);
8686	n/a	return resunicode;
8687	n/a	}
8688	n/a
8689	n/a	/* Lookup the character ch in the mapping and put the result in result,
8690	n/a	which must be decrefed by the caller.
8691	n/a	Return 0 on success, -1 on error */
8692	n/a	static int
8693	n/a	charmaptranslate_lookup(Py_UCS4 c, PyObject mapping, PyObject *result)
8694	n/a	{
8695	n/a	PyObject *w = PyLong_FromLong((long)c);
8696	n/a	PyObject *x;
8697	n/a