Python code coverage for Modules/cjkcodecs/_codecs_cn.c

#	count	content
1	n/a	/*
2	n/a	* _codecs_cn.c: Codecs collection for Mainland Chinese encodings
3	n/a	*
4	n/a	* Written by Hye-Shik Chang <perky@FreeBSD.org>
5	n/a	*/
6	n/a
7	n/a	#include "cjkcodecs.h"
8	n/a	#include "mappings_cn.h"
9	n/a
10	n/a	/**
11	n/a	* hz is predefined as 100 on AIX. So we undefine it to avoid
12	n/a	* conflict against hz codec's.
13	n/a	*/
14	n/a	#ifdef _AIX
15	n/a	#undef hz
16	n/a	#endif
17	n/a
18	n/a	/* GBK and GB2312 map differently in few code points that are listed below:
19	n/a	*
20	n/a	* gb2312 gbk
21	n/a	* A1A4 U+30FB KATAKANA MIDDLE DOT U+00B7 MIDDLE DOT
22	n/a	* A1AA U+2015 HORIZONTAL BAR U+2014 EM DASH
23	n/a	* A844 undefined U+2015 HORIZONTAL BAR
24	n/a	*/
25	n/a
26	n/a	#define GBK_DECODE(dc1, dc2, writer) \
27	n/a	if ((dc1) == 0xa1 && (dc2) == 0xaa) { \
28	n/a	OUTCHAR(0x2014); \
29	n/a	} \
30	n/a	else if ((dc1) == 0xa8 && (dc2) == 0x44) { \
31	n/a	OUTCHAR(0x2015); \
32	n/a	} \
33	n/a	else if ((dc1) == 0xa1 && (dc2) == 0xa4) { \
34	n/a	OUTCHAR(0x00b7); \
35	n/a	} \
36	n/a	else if (TRYMAP_DEC(gb2312, decoded, dc1 ^ 0x80, dc2 ^ 0x80)) { \
37	n/a	OUTCHAR(decoded); \
38	n/a	} \
39	n/a	else if (TRYMAP_DEC(gbkext, decoded, dc1, dc2)) { \
40	n/a	OUTCHAR(decoded); \
41	n/a	}
42	n/a
43	n/a	#define GBK_ENCODE(code, assi) \
44	n/a	if ((code) == 0x2014) { \
45	n/a	(assi) = 0xa1aa; \
46	n/a	} else if ((code) == 0x2015) { \
47	n/a	(assi) = 0xa844; \
48	n/a	} else if ((code) == 0x00b7) { \
49	n/a	(assi) = 0xa1a4; \
50	n/a	} else if ((code) != 0x30fb && TRYMAP_ENC(gbcommon, assi, code)) { \
51	n/a	; \
52	n/a	}
53	n/a
54	n/a	/*
55	n/a	* GB2312 codec
56	n/a	*/
57	n/a
58	n/a	ENCODER(gb2312)
59	n/a	{
60	n/a	while (*inpos < inlen) {
61	n/a	Py_UCS4 c = INCHAR1;
62	n/a	DBCHAR code;
63	n/a
64	n/a	if (c < 0x80) {
65	n/a	WRITEBYTE1((unsigned char)c);
66	n/a	NEXT(1, 1);
67	n/a	continue;
68	n/a	}
69	n/a
70	n/a	if (c > 0xFFFF)
71	n/a	return 1;
72	n/a
73	n/a	REQUIRE_OUTBUF(2);
74	n/a	if (TRYMAP_ENC(gbcommon, code, c))
75	n/a	;
76	n/a	else
77	n/a	return 1;
78	n/a
79	n/a	if (code & 0x8000) /* MSB set: GBK */
80	n/a	return 1;
81	n/a
82	n/a	OUTBYTE1((code >> 8) \| 0x80);
83	n/a	OUTBYTE2((code & 0xFF) \| 0x80);
84	n/a	NEXT(1, 2);
85	n/a	}
86	n/a
87	n/a	return 0;
88	n/a	}
89	n/a
90	n/a	DECODER(gb2312)
91	n/a	{
92	n/a	while (inleft > 0) {
93	n/a	unsigned char c = **inbuf;
94	n/a	Py_UCS4 decoded;
95	n/a
96	n/a	if (c < 0x80) {
97	n/a	OUTCHAR(c);
98	n/a	NEXT_IN(1);
99	n/a	continue;
100	n/a	}
101	n/a
102	n/a	REQUIRE_INBUF(2);
103	n/a	if (TRYMAP_DEC(gb2312, decoded, c ^ 0x80, INBYTE2 ^ 0x80)) {
104	n/a	OUTCHAR(decoded);
105	n/a	NEXT_IN(2);
106	n/a	}
107	n/a	else
108	n/a	return 1;
109	n/a	}
110	n/a
111	n/a	return 0;
112	n/a	}
113	n/a
114	n/a
115	n/a	/*
116	n/a	* GBK codec
117	n/a	*/
118	n/a
119	n/a	ENCODER(gbk)
120	n/a	{
121	n/a	while (*inpos < inlen) {
122	n/a	Py_UCS4 c = INCHAR1;
123	n/a	DBCHAR code;
124	n/a
125	n/a	if (c < 0x80) {
126	n/a	WRITEBYTE1((unsigned char)c);
127	n/a	NEXT(1, 1);
128	n/a	continue;
129	n/a	}
130	n/a
131	n/a	if (c > 0xFFFF)
132	n/a	return 1;
133	n/a
134	n/a	REQUIRE_OUTBUF(2);
135	n/a
136	n/a	GBK_ENCODE(c, code)
137	n/a	else
138	n/a	return 1;
139	n/a
140	n/a	OUTBYTE1((code >> 8) \| 0x80);
141	n/a	if (code & 0x8000)
142	n/a	OUTBYTE2((code & 0xFF)); /* MSB set: GBK */
143	n/a	else
144	n/a	OUTBYTE2((code & 0xFF) \| 0x80); /* MSB unset: GB2312 */
145	n/a	NEXT(1, 2);
146	n/a	}
147	n/a
148	n/a	return 0;
149	n/a	}
150	n/a
151	n/a	DECODER(gbk)
152	n/a	{
153	n/a	while (inleft > 0) {
154	n/a	unsigned char c = INBYTE1;
155	n/a	Py_UCS4 decoded;
156	n/a
157	n/a	if (c < 0x80) {
158	n/a	OUTCHAR(c);
159	n/a	NEXT_IN(1);
160	n/a	continue;
161	n/a	}
162	n/a
163	n/a	REQUIRE_INBUF(2);
164	n/a
165	n/a	GBK_DECODE(c, INBYTE2, writer)
166	n/a	else
167	n/a	return 1;
168	n/a
169	n/a	NEXT_IN(2);
170	n/a	}
171	n/a
172	n/a	return 0;
173	n/a	}
174	n/a
175	n/a
176	n/a	/*
177	n/a	* GB18030 codec
178	n/a	*/
179	n/a
180	n/a	ENCODER(gb18030)
181	n/a	{
182	n/a	while (*inpos < inlen) {
183	n/a	Py_UCS4 c = INCHAR1;
184	n/a	DBCHAR code;
185	n/a
186	n/a	if (c < 0x80) {
187	n/a	WRITEBYTE1(c);
188	n/a	NEXT(1, 1);
189	n/a	continue;
190	n/a	}
191	n/a
192	n/a	if (c >= 0x10000) {
193	n/a	Py_UCS4 tc = c - 0x10000;
194	n/a	assert (c <= 0x10FFFF);
195	n/a
196	n/a	REQUIRE_OUTBUF(4);
197	n/a
198	n/a	OUTBYTE4((unsigned char)(tc % 10) + 0x30);
199	n/a	tc /= 10;
200	n/a	OUTBYTE3((unsigned char)(tc % 126) + 0x81);
201	n/a	tc /= 126;
202	n/a	OUTBYTE2((unsigned char)(tc % 10) + 0x30);
203	n/a	tc /= 10;
204	n/a	OUTBYTE1((unsigned char)(tc + 0x90));
205	n/a
206	n/a	NEXT(1, 4);
207	n/a	continue;
208	n/a	}
209	n/a
210	n/a	REQUIRE_OUTBUF(2);
211	n/a
212	n/a	GBK_ENCODE(c, code)
213	n/a	else if (TRYMAP_ENC(gb18030ext, code, c))
214	n/a	;
215	n/a	else {
216	n/a	const struct _gb18030_to_unibmp_ranges *utrrange;
217	n/a
218	n/a	REQUIRE_OUTBUF(4);
219	n/a
220	n/a	for (utrrange = gb18030_to_unibmp_ranges;
221	n/a	utrrange->first != 0;
222	n/a	utrrange++)
223	n/a	if (utrrange->first <= c &&
224	n/a	c <= utrrange->last) {
225	n/a	Py_UCS4 tc;
226	n/a
227	n/a	tc = c - utrrange->first +
228	n/a	utrrange->base;
229	n/a
230	n/a	OUTBYTE4((unsigned char)(tc % 10) + 0x30);
231	n/a	tc /= 10;
232	n/a	OUTBYTE3((unsigned char)(tc % 126) + 0x81);
233	n/a	tc /= 126;
234	n/a	OUTBYTE2((unsigned char)(tc % 10) + 0x30);
235	n/a	tc /= 10;
236	n/a	OUTBYTE1((unsigned char)tc + 0x81);
237	n/a
238	n/a	NEXT(1, 4);
239	n/a	break;
240	n/a	}
241	n/a
242	n/a	if (utrrange->first == 0)
243	n/a	return 1;
244	n/a	continue;
245	n/a	}
246	n/a
247	n/a	OUTBYTE1((code >> 8) \| 0x80);
248	n/a	if (code & 0x8000)
249	n/a	OUTBYTE2((code & 0xFF)); /* MSB set: GBK or GB18030ext */
250	n/a	else
251	n/a	OUTBYTE2((code & 0xFF) \| 0x80); /* MSB unset: GB2312 */
252	n/a
253	n/a	NEXT(1, 2);
254	n/a	}
255	n/a
256	n/a	return 0;
257	n/a	}
258	n/a
259	n/a	DECODER(gb18030)
260	n/a	{
261	n/a	while (inleft > 0) {
262	n/a	unsigned char c = INBYTE1, c2;
263	n/a	Py_UCS4 decoded;
264	n/a
265	n/a	if (c < 0x80) {
266	n/a	OUTCHAR(c);
267	n/a	NEXT_IN(1);
268	n/a	continue;
269	n/a	}
270	n/a
271	n/a	REQUIRE_INBUF(2);
272	n/a
273	n/a	c2 = INBYTE2;
274	n/a	if (c2 >= 0x30 && c2 <= 0x39) { /* 4 bytes seq */
275	n/a	const struct _gb18030_to_unibmp_ranges *utr;
276	n/a	unsigned char c3, c4;
277	n/a	Py_UCS4 lseq;
278	n/a
279	n/a	REQUIRE_INBUF(4);
280	n/a	c3 = INBYTE3;
281	n/a	c4 = INBYTE4;
282	n/a	if (c < 0x81 \|\| c3 < 0x81 \|\| c4 < 0x30 \|\| c4 > 0x39)
283	n/a	return 1;
284	n/a	c -= 0x81; c2 -= 0x30;
285	n/a	c3 -= 0x81; c4 -= 0x30;
286	n/a
287	n/a	if (c < 4) { /* U+0080 - U+FFFF */
288	n/a	lseq = ((Py_UCS4)c * 10 + c2) * 1260 +
289	n/a	(Py_UCS4)c3 * 10 + c4;
290	n/a	if (lseq < 39420) {
291	n/a	for (utr = gb18030_to_unibmp_ranges;
292	n/a	lseq >= (utr + 1)->base;
293	n/a	utr++) ;
294	n/a	OUTCHAR(utr->first - utr->base + lseq);
295	n/a	NEXT_IN(4);
296	n/a	continue;
297	n/a	}
298	n/a	}
299	n/a	else if (c >= 15) { /* U+10000 - U+10FFFF */
300	n/a	lseq = 0x10000 + (((Py_UCS4)c-15) * 10 + c2)
301	n/a	* 1260 + (Py_UCS4)c3 * 10 + c4;
302	n/a	if (lseq <= 0x10FFFF) {
303	n/a	OUTCHAR(lseq);
304	n/a	NEXT_IN(4);
305	n/a	continue;
306	n/a	}
307	n/a	}
308	n/a	return 1;
309	n/a	}
310	n/a
311	n/a	GBK_DECODE(c, c2, writer)
312	n/a	else if (TRYMAP_DEC(gb18030ext, decoded, c, c2))
313	n/a	OUTCHAR(decoded);
314	n/a	else
315	n/a	return 1;
316	n/a
317	n/a	NEXT_IN(2);
318	n/a	}
319	n/a
320	n/a	return 0;
321	n/a	}
322	n/a
323	n/a
324	n/a	/*
325	n/a	* HZ codec
326	n/a	*/
327	n/a
328	n/a	ENCODER_INIT(hz)
329	n/a	{
330	n/a	state->i = 0;
331	n/a	return 0;
332	n/a	}
333	n/a
334	n/a	ENCODER_RESET(hz)
335	n/a	{
336	n/a	if (state->i != 0) {
337	n/a	WRITEBYTE2('~', '}');
338	n/a	state->i = 0;
339	n/a	NEXT_OUT(2);
340	n/a	}
341	n/a	return 0;
342	n/a	}
343	n/a
344	n/a	ENCODER(hz)
345	n/a	{
346	n/a	while (*inpos < inlen) {
347	n/a	Py_UCS4 c = INCHAR1;
348	n/a	DBCHAR code;
349	n/a
350	n/a	if (c < 0x80) {
351	n/a	if (state->i == 0) {
352	n/a	WRITEBYTE1((unsigned char)c);
353	n/a	NEXT(1, 1);
354	n/a	}
355	n/a	else {
356	n/a	WRITEBYTE3('~', '}', (unsigned char)c);
357	n/a	NEXT(1, 3);
358	n/a	state->i = 0;
359	n/a	}
360	n/a	continue;
361	n/a	}
362	n/a
363	n/a	if (c > 0xFFFF)
364	n/a	return 1;
365	n/a
366	n/a	if (TRYMAP_ENC(gbcommon, code, c))
367	n/a	;
368	n/a	else
369	n/a	return 1;
370	n/a
371	n/a	if (code & 0x8000) /* MSB set: GBK */
372	n/a	return 1;
373	n/a
374	n/a	if (state->i == 0) {
375	n/a	WRITEBYTE4('~', '{', code >> 8, code & 0xff);
376	n/a	NEXT(1, 4);
377	n/a	state->i = 1;
378	n/a	}
379	n/a	else {
380	n/a	WRITEBYTE2(code >> 8, code & 0xff);
381	n/a	NEXT(1, 2);
382	n/a	}
383	n/a	}
384	n/a
385	n/a	return 0;
386	n/a	}
387	n/a
388	n/a	DECODER_INIT(hz)
389	n/a	{
390	n/a	state->i = 0;
391	n/a	return 0;
392	n/a	}
393	n/a
394	n/a	DECODER_RESET(hz)
395	n/a	{
396	n/a	state->i = 0;
397	n/a	return 0;
398	n/a	}
399	n/a
400	n/a	DECODER(hz)
401	n/a	{
402	n/a	while (inleft > 0) {
403	n/a	unsigned char c = INBYTE1;
404	n/a	Py_UCS4 decoded;
405	n/a
406	n/a	if (c == '~') {
407	n/a	unsigned char c2 = INBYTE2;
408	n/a
409	n/a	REQUIRE_INBUF(2);
410	n/a	if (c2 == '~') {
411	n/a	OUTCHAR('~');
412	n/a	NEXT_IN(2);
413	n/a	continue;
414	n/a	}
415	n/a	else if (c2 == '{' && state->i == 0)
416	n/a	state->i = 1; /* set GB */
417	n/a	else if (c2 == '}' && state->i == 1)
418	n/a	state->i = 0; /* set ASCII */
419	n/a	else if (c2 == '\n')
420	n/a	; /* line-continuation */
421	n/a	else
422	n/a	return 1;
423	n/a	NEXT_IN(2);
424	n/a	continue;
425	n/a	}
426	n/a
427	n/a	if (c & 0x80)
428	n/a	return 1;
429	n/a
430	n/a	if (state->i == 0) { /* ASCII mode */
431	n/a	OUTCHAR(c);
432	n/a	NEXT_IN(1);
433	n/a	}
434	n/a	else { /* GB mode */
435	n/a	REQUIRE_INBUF(2);
436	n/a	if (TRYMAP_DEC(gb2312, decoded, c, INBYTE2)) {
437	n/a	OUTCHAR(decoded);
438	n/a	NEXT_IN(2);
439	n/a	}
440	n/a	else
441	n/a	return 1;
442	n/a	}
443	n/a	}
444	n/a
445	n/a	return 0;
446	n/a	}
447	n/a
448	n/a
449	n/a	BEGIN_MAPPINGS_LIST
450	n/a	MAPPING_DECONLY(gb2312)
451	n/a	MAPPING_DECONLY(gbkext)
452	n/a	MAPPING_ENCONLY(gbcommon)
453	n/a	MAPPING_ENCDEC(gb18030ext)
454	n/a	END_MAPPINGS_LIST
455	n/a
456	n/a	BEGIN_CODECS_LIST
457	n/a	CODEC_STATELESS(gb2312)
458	n/a	CODEC_STATELESS(gbk)
459	n/a	CODEC_STATELESS(gb18030)
460	n/a	CODEC_STATEFUL(hz)
461	n/a	END_CODECS_LIST
462	n/a
463	n/a	I_AM_A_MODULE_FOR(cn)