Python code coverage for Tools/unicode/gencodec.py

#	count	content
1	n/a	""" Unicode Mapping Parser and Codec Generator.
2	n/a
3	n/a	This script parses Unicode mapping files as available from the Unicode
4	n/a	site (ftp://ftp.unicode.org/Public/MAPPINGS/) and creates Python codec
5	n/a	modules from them. The codecs use the standard character mapping codec
6	n/a	to actually apply the mapping.
7	n/a
8	n/a	Synopsis: gencodec.py dir codec_prefix
9	n/a
10	n/a	All files in dir are scanned and those producing non-empty mappings
11	n/a	will be written to <codec_prefix><mapname>.py with <mapname> being the
12	n/a	first part of the map's filename ('a' in a.b.c.txt) converted to
13	n/a	lowercase with hyphens replaced by underscores.
14	n/a
15	n/a	The tool also writes marshalled versions of the mapping tables to the
16	n/a	same location (with .mapping extension).
17	n/a
18	n/a	Written by Marc-Andre Lemburg (mal@lemburg.com).
19	n/a
20	n/a	(c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
21	n/a	(c) Copyright Guido van Rossum, 2000.
22	n/a
23	n/a	Table generation:
24	n/a	(c) Copyright Marc-Andre Lemburg, 2005.
25	n/a	Licensed to PSF under a Contributor Agreement.
26	n/a
27	n/a	"""#"
28	n/a
29	n/a	import re, os, marshal, codecs
30	n/a
31	n/a	# Maximum allowed size of charmap tables
32	n/a	MAX_TABLE_SIZE = 8192
33	n/a
34	n/a	# Standard undefined Unicode code point
35	n/a	UNI_UNDEFINED = chr(0xFFFE)
36	n/a
37	n/a	# Placeholder for a missing code point
38	n/a	MISSING_CODE = -1
39	n/a
40	n/a	mapRE = re.compile(r'((?:0x[0-9a-fA-F]+\+?)+)'
41	n/a	r'\s+'
42	n/a	r'((?:(?:0x[0-9a-fA-Z]+\|<[A-Za-z]+>)\+?)*)'
43	n/a	r'\s*'
44	n/a	r'(#.+)?')
45	n/a
46	n/a	def parsecodes(codes, len=len, range=range):
47	n/a
48	n/a	""" Converts code combinations to either a single code integer
49	n/a	or a tuple of integers.
50	n/a
51	n/a	meta-codes (in angular brackets, e.g. <LR> and <RL>) are
52	n/a	ignored.
53	n/a
54	n/a	Empty codes or illegal ones are returned as None.
55	n/a
56	n/a	"""
57	n/a	if not codes:
58	n/a	return MISSING_CODE
59	n/a	l = codes.split('+')
60	n/a	if len(l) == 1:
61	n/a	return int(l[0],16)
62	n/a	for i in range(len(l)):
63	n/a	try:
64	n/a	l[i] = int(l[i],16)
65	n/a	except ValueError:
66	n/a	l[i] = MISSING_CODE
67	n/a	l = [x for x in l if x != MISSING_CODE]
68	n/a	if len(l) == 1:
69	n/a	return l[0]
70	n/a	else:
71	n/a	return tuple(l)
72	n/a
73	n/a	def readmap(filename):
74	n/a
75	n/a	f = open(filename,'r')
76	n/a	lines = f.readlines()
77	n/a	f.close()
78	n/a	enc2uni = {}
79	n/a	identity = []
80	n/a	unmapped = list(range(256))
81	n/a
82	n/a	# UTC mapping tables per convention don't include the identity
83	n/a	# mappings for code points 0x00 - 0x1F and 0x7F, unless these are
84	n/a	# explicitly mapped to different characters or undefined
85	n/a	for i in list(range(32)) + [127]:
86	n/a	identity.append(i)
87	n/a	unmapped.remove(i)
88	n/a	enc2uni[i] = (i, 'CONTROL CHARACTER')
89	n/a
90	n/a	for line in lines:
91	n/a	line = line.strip()
92	n/a	if not line or line[0] == '#':
93	n/a	continue
94	n/a	m = mapRE.match(line)
95	n/a	if not m:
96	n/a	#print '* not matched: %s' % repr(line)
97	n/a	continue
98	n/a	enc,uni,comment = m.groups()
99	n/a	enc = parsecodes(enc)
100	n/a	uni = parsecodes(uni)
101	n/a	if comment is None:
102	n/a	comment = ''
103	n/a	else:
104	n/a	comment = comment[1:].strip()
105	n/a	if not isinstance(enc, tuple) and enc < 256:
106	n/a	if enc in unmapped:
107	n/a	unmapped.remove(enc)
108	n/a	if enc == uni:
109	n/a	identity.append(enc)
110	n/a	enc2uni[enc] = (uni,comment)
111	n/a	else:
112	n/a	enc2uni[enc] = (uni,comment)
113	n/a
114	n/a	# If there are more identity-mapped entries than unmapped entries,
115	n/a	# it pays to generate an identity dictionary first, and add explicit
116	n/a	# mappings to None for the rest
117	n/a	if len(identity) >= len(unmapped):
118	n/a	for enc in unmapped:
119	n/a	enc2uni[enc] = (MISSING_CODE, "")
120	n/a	enc2uni['IDENTITY'] = 256
121	n/a
122	n/a	return enc2uni
123	n/a
124	n/a	def hexrepr(t, precision=4):
125	n/a
126	n/a	if t is None:
127	n/a	return 'None'
128	n/a	try:
129	n/a	len(t)
130	n/a	except TypeError:
131	n/a	return '0x%0*X' % (precision, t)
132	n/a	try:
133	n/a	return '(' + ', '.join(['0x%0*X' % (precision, item)
134	n/a	for item in t]) + ')'
135	n/a	except TypeError as why:
136	n/a	print('* failed to convert %r: %s' % (t, why))
137	n/a	raise
138	n/a
139	n/a	def python_mapdef_code(varname, map, comments=1, precisions=(2, 4)):
140	n/a
141	n/a	l = []
142	n/a	append = l.append
143	n/a	if "IDENTITY" in map:
144	n/a	append("%s = codecs.make_identity_dict(range(%d))" %
145	n/a	(varname, map["IDENTITY"]))
146	n/a	append("%s.update({" % varname)
147	n/a	splits = 1
148	n/a	del map["IDENTITY"]
149	n/a	identity = 1
150	n/a	else:
151	n/a	append("%s = {" % varname)
152	n/a	splits = 0
153	n/a	identity = 0
154	n/a
155	n/a	mappings = sorted(map.items())
156	n/a	i = 0
157	n/a	key_precision, value_precision = precisions
158	n/a	for mapkey, mapvalue in mappings:
159	n/a	mapcomment = ''
160	n/a	if isinstance(mapkey, tuple):
161	n/a	(mapkey, mapcomment) = mapkey
162	n/a	if isinstance(mapvalue, tuple):
163	n/a	(mapvalue, mapcomment) = mapvalue
164	n/a	if mapkey is None:
165	n/a	continue
166	n/a	if (identity and
167	n/a	mapkey == mapvalue and
168	n/a	mapkey < 256):
169	n/a	# No need to include identity mappings, since these
170	n/a	# are already set for the first 256 code points.
171	n/a	continue
172	n/a	key = hexrepr(mapkey, key_precision)
173	n/a	value = hexrepr(mapvalue, value_precision)
174	n/a	if mapcomment and comments:
175	n/a	append(' %s: %s,\t# %s' % (key, value, mapcomment))
176	n/a	else:
177	n/a	append(' %s: %s,' % (key, value))
178	n/a	i += 1
179	n/a	if i == 4096:
180	n/a	# Split the definition into parts to that the Python
181	n/a	# parser doesn't dump core
182	n/a	if splits == 0:
183	n/a	append('}')
184	n/a	else:
185	n/a	append('})')
186	n/a	append('%s.update({' % varname)
187	n/a	i = 0
188	n/a	splits = splits + 1
189	n/a	if splits == 0:
190	n/a	append('}')
191	n/a	else:
192	n/a	append('})')
193	n/a
194	n/a	return l
195	n/a
196	n/a	def python_tabledef_code(varname, map, comments=1, key_precision=2):
197	n/a
198	n/a	l = []
199	n/a	append = l.append
200	n/a	append('%s = (' % varname)
201	n/a
202	n/a	# Analyze map and create table dict
203	n/a	mappings = sorted(map.items())
204	n/a	table = {}
205	n/a	maxkey = 255
206	n/a	if 'IDENTITY' in map:
207	n/a	for key in range(256):
208	n/a	table[key] = (key, '')
209	n/a	del map['IDENTITY']
210	n/a	for mapkey, mapvalue in mappings:
211	n/a	mapcomment = ''
212	n/a	if isinstance(mapkey, tuple):
213	n/a	(mapkey, mapcomment) = mapkey
214	n/a	if isinstance(mapvalue, tuple):
215	n/a	(mapvalue, mapcomment) = mapvalue
216	n/a	if mapkey == MISSING_CODE:
217	n/a	continue
218	n/a	table[mapkey] = (mapvalue, mapcomment)
219	n/a	if mapkey > maxkey:
220	n/a	maxkey = mapkey
221	n/a	if maxkey > MAX_TABLE_SIZE:
222	n/a	# Table too large
223	n/a	return None
224	n/a
225	n/a	# Create table code
226	n/a	maxchar = 0
227	n/a	for key in range(maxkey + 1):
228	n/a	if key not in table:
229	n/a	mapvalue = MISSING_CODE
230	n/a	mapcomment = 'UNDEFINED'
231	n/a	else:
232	n/a	mapvalue, mapcomment = table[key]
233	n/a	if mapvalue == MISSING_CODE:
234	n/a	mapchar = UNI_UNDEFINED
235	n/a	else:
236	n/a	if isinstance(mapvalue, tuple):
237	n/a	# 1-n mappings not supported
238	n/a	return None
239	n/a	else:
240	n/a	mapchar = chr(mapvalue)
241	n/a	maxchar = max(maxchar, ord(mapchar))
242	n/a	if mapcomment and comments:
243	n/a	append(' %a \t# %s -> %s' % (mapchar,
244	n/a	hexrepr(key, key_precision),
245	n/a	mapcomment))
246	n/a	else:
247	n/a	append(' %a' % mapchar)
248	n/a
249	n/a	if maxchar < 256:
250	n/a	append(' %a \t## Widen to UCS2 for optimization' % UNI_UNDEFINED)
251	n/a	append(')')
252	n/a	return l
253	n/a
254	n/a	def codegen(name, map, encodingname, comments=1):
255	n/a
256	n/a	""" Returns Python source for the given map.
257	n/a
258	n/a	Comments are included in the source, if comments is true (default).
259	n/a
260	n/a	"""
261	n/a	# Generate code
262	n/a	decoding_map_code = python_mapdef_code(
263	n/a	'decoding_map',
264	n/a	map,
265	n/a	comments=comments)
266	n/a	decoding_table_code = python_tabledef_code(
267	n/a	'decoding_table',
268	n/a	map,
269	n/a	comments=comments)
270	n/a	encoding_map_code = python_mapdef_code(
271	n/a	'encoding_map',
272	n/a	codecs.make_encoding_map(map),
273	n/a	comments=comments,
274	n/a	precisions=(4, 2))
275	n/a
276	n/a	if decoding_table_code:
277	n/a	suffix = 'table'
278	n/a	else:
279	n/a	suffix = 'map'
280	n/a
281	n/a	l = [
282	n/a	'''\
283	n/a	""" Python Character Mapping Codec %s generated from '%s' with gencodec.py.
284	n/a
285	n/a	"""#"
286	n/a
287	n/a	import codecs
288	n/a
289	n/a	### Codec APIs
290	n/a
291	n/a	class Codec(codecs.Codec):
292	n/a
293	n/a	def encode(self, input, errors='strict'):
294	n/a	return codecs.charmap_encode(input, errors, encoding_%s)
295	n/a
296	n/a	def decode(self, input, errors='strict'):
297	n/a	return codecs.charmap_decode(input, errors, decoding_%s)
298	n/a	''' % (encodingname, name, suffix, suffix)]
299	n/a	l.append('''\
300	n/a	class IncrementalEncoder(codecs.IncrementalEncoder):
301	n/a	def encode(self, input, final=False):
302	n/a	return codecs.charmap_encode(input, self.errors, encoding_%s)[0]
303	n/a
304	n/a	class IncrementalDecoder(codecs.IncrementalDecoder):
305	n/a	def decode(self, input, final=False):
306	n/a	return codecs.charmap_decode(input, self.errors, decoding_%s)[0]''' %
307	n/a	(suffix, suffix))
308	n/a
309	n/a	l.append('''
310	n/a	class StreamWriter(Codec, codecs.StreamWriter):
311	n/a	pass
312	n/a
313	n/a	class StreamReader(Codec, codecs.StreamReader):
314	n/a	pass
315	n/a
316	n/a	### encodings module API
317	n/a
318	n/a	def getregentry():
319	n/a	return codecs.CodecInfo(
320	n/a	name=%r,
321	n/a	encode=Codec().encode,
322	n/a	decode=Codec().decode,
323	n/a	incrementalencoder=IncrementalEncoder,
324	n/a	incrementaldecoder=IncrementalDecoder,
325	n/a	streamreader=StreamReader,
326	n/a	streamwriter=StreamWriter,
327	n/a	)
328	n/a	''' % encodingname.replace('_', '-'))
329	n/a
330	n/a	# Add decoding table or map (with preference to the table)
331	n/a	if not decoding_table_code:
332	n/a	l.append('''
333	n/a	### Decoding Map
334	n/a	''')
335	n/a	l.extend(decoding_map_code)
336	n/a	else:
337	n/a	l.append('''
338	n/a	### Decoding Table
339	n/a	''')
340	n/a	l.extend(decoding_table_code)
341	n/a
342	n/a	# Add encoding map
343	n/a	if decoding_table_code:
344	n/a	l.append('''
345	n/a	### Encoding table
346	n/a	encoding_table = codecs.charmap_build(decoding_table)
347	n/a	''')
348	n/a	else:
349	n/a	l.append('''
350	n/a	### Encoding Map
351	n/a	''')
352	n/a	l.extend(encoding_map_code)
353	n/a
354	n/a	# Final new-line
355	n/a	l.append('')
356	n/a
357	n/a	return '\n'.join(l).expandtabs()
358	n/a
359	n/a	def pymap(name,map,pyfile,encodingname,comments=1):
360	n/a
361	n/a	code = codegen(name,map,encodingname,comments)
362	n/a	f = open(pyfile,'w')
363	n/a	f.write(code)
364	n/a	f.close()
365	n/a
366	n/a	def marshalmap(name,map,marshalfile):
367	n/a
368	n/a	d = {}
369	n/a	for e,(u,c) in map.items():
370	n/a	d[e] = (u,c)
371	n/a	f = open(marshalfile,'wb')
372	n/a	marshal.dump(d,f)
373	n/a	f.close()
374	n/a
375	n/a	def convertdir(dir, dirprefix='', nameprefix='', comments=1):
376	n/a
377	n/a	mapnames = os.listdir(dir)
378	n/a	for mapname in mapnames:
379	n/a	mappathname = os.path.join(dir, mapname)
380	n/a	if not os.path.isfile(mappathname):
381	n/a	continue
382	n/a	name = os.path.split(mapname)[1]
383	n/a	name = name.replace('-','_')
384	n/a	name = name.split('.')[0]
385	n/a	name = name.lower()
386	n/a	name = nameprefix + name
387	n/a	codefile = name + '.py'
388	n/a	marshalfile = name + '.mapping'
389	n/a	print('converting %s to %s and %s' % (mapname,
390	n/a	dirprefix + codefile,
391	n/a	dirprefix + marshalfile))
392	n/a	try:
393	n/a	map = readmap(os.path.join(dir,mapname))
394	n/a	if not map:
395	n/a	print('* map is empty; skipping')
396	n/a	else:
397	n/a	pymap(mappathname, map, dirprefix + codefile,name,comments)
398	n/a	marshalmap(mappathname, map, dirprefix + marshalfile)
399	n/a	except ValueError as why:
400	n/a	print('* conversion failed: %s' % why)
401	n/a	raise
402	n/a
403	n/a	def rewritepythondir(dir, dirprefix='', comments=1):
404	n/a
405	n/a	mapnames = os.listdir(dir)
406	n/a	for mapname in mapnames:
407	n/a	if not mapname.endswith('.mapping'):
408	n/a	continue
409	n/a	name = mapname[:-len('.mapping')]
410	n/a	codefile = name + '.py'
411	n/a	print('converting %s to %s' % (mapname,
412	n/a	dirprefix + codefile))
413	n/a	try:
414	n/a	map = marshal.load(open(os.path.join(dir,mapname),
415	n/a	'rb'))
416	n/a	if not map:
417	n/a	print('* map is empty; skipping')
418	n/a	else:
419	n/a	pymap(mapname, map, dirprefix + codefile,name,comments)
420	n/a	except ValueError as why:
421	n/a	print('* conversion failed: %s' % why)
422	n/a
423	n/a	if __name__ == '__main__':
424	n/a
425	n/a	import sys
426	n/a	if 1:
427	n/a	convertdir(*sys.argv[1:])
428	n/a	else:
429	n/a	rewritepythondir(*sys.argv[1:])