Python code coverage for Lib/compiler/symbols.py

#	count	content
1	2	"""Module symbol-table generator"""
2	n/a
3	2	from compiler import ast
4	2	from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICT, \
5	n/a	SC_FREE, SC_CELL, SC_UNKNOWN
6	2	from compiler.misc import mangle
7	2	import types
8	n/a
9	n/a
10	2	import sys
11	n/a
12	2	MANGLE_LEN = 256
13	n/a
14	4	class Scope:
15	n/a	# XXX how much information do I need about each name?
16	2	def __init__(self, name, module, klass=None):
17	1059	self.name = name
18	1059	self.module = module
19	1059	self.defs = {}
20	1059	self.uses = {}
21	1059	self.globals = {}
22	1059	self.params = {}
23	1059	self.frees = {}
24	1059	self.cells = {}
25	1059	self.children = []
26	n/a	# nested is true if the class could contain free variables,
27	n/a	# i.e. if it is nested within another function.
28	1059	self.nested = None
29	1059	self.generator = None
30	1059	self.klass = None
31	1059	if klass is not None:
32	993	for i in range(len(klass)):
33	993	if klass[i] != '_':
34	976	self.klass = klass[i:]
35	976	break
36	n/a
37	2	def __repr__(self):
38	0	return "<%s: %s>" % (self.__class__.__name__, self.name)
39	n/a
40	2	def mangle(self, name):
41	10098	if self.klass is None:
42	619	return name
43	9479	return mangle(name, self.klass)
44	n/a
45	2	def add_def(self, name):
46	2327	self.defs[self.mangle(name)] = 1
47	n/a
48	2	def add_use(self, name):
49	6815	self.uses[self.mangle(name)] = 1
50	n/a
51	2	def add_global(self, name):
52	11	name = self.mangle(name)
53	11	if name in self.uses or name in self.defs:
54	0	pass # XXX warn about global following def/use
55	11	if name in self.params:
56	0	raise SyntaxError, "%s in %s is global and parameter" % \
57	0	(name, self.name)
58	11	self.globals[name] = 1
59	11	self.module.add_def(name)
60	n/a
61	2	def add_param(self, name):
62	945	name = self.mangle(name)
63	945	self.defs[name] = 1
64	945	self.params[name] = 1
65	n/a
66	2	def get_names(self):
67	0	d = {}
68	0	d.update(self.defs)
69	0	d.update(self.uses)
70	0	d.update(self.globals)
71	0	return d.keys()
72	n/a
73	2	def add_child(self, child):
74	1028	self.children.append(child)
75	n/a
76	2	def get_children(self):
77	0	return self.children
78	n/a
79	2	def DEBUG(self):
80	0	print >> sys.stderr, self.name, self.nested and "nested" or ""
81	0	print >> sys.stderr, "\tglobals: ", self.globals
82	0	print >> sys.stderr, "\tcells: ", self.cells
83	0	print >> sys.stderr, "\tdefs: ", self.defs
84	0	print >> sys.stderr, "\tuses: ", self.uses
85	0	print >> sys.stderr, "\tfrees:", self.frees
86	n/a
87	2	def check_name(self, name):
88	n/a	"""Return scope of name.
89	n/a
90	n/a	The scope of a name could be LOCAL, GLOBAL, FREE, or CELL.
91	n/a	"""
92	9971	if name in self.globals:
93	301	return SC_GLOBAL_EXPLICT
94	9670	if name in self.cells:
95	385	return SC_CELL
96	9285	if name in self.defs:
97	7042	return SC_LOCAL
98	2243	if self.nested and (name in self.frees or name in self.uses):
99	314	return SC_FREE
100	1929	if self.nested:
101	365	return SC_UNKNOWN
102	n/a	else:
103	1564	return SC_GLOBAL_IMPLICIT
104	n/a
105	2	def get_free_vars(self):
106	3010	if not self.nested:
107	931	return ()
108	2079	free = {}
109	2079	free.update(self.frees)
110	4425	for name in self.uses.keys():
111	2346	if name not in self.defs and name not in self.globals:
112	554	free[name] = 1
113	2079	return free.keys()
114	n/a
115	2	def handle_children(self):
116	1982	for child in self.children:
117	954	frees = child.get_free_vars()
118	954	globals = self.add_frees(frees)
119	1136	for name in globals:
120	182	child.force_global(name)
121	n/a
122	2	def force_global(self, name):
123	n/a	"""Force name to be global in scope.
124	n/a
125	n/a	Some child of the current node had a free reference to name.
126	n/a	When the child was processed, it was labelled a free
127	n/a	variable. Now that all its enclosing scope have been
128	n/a	processed, the name is known to be a global or builtin. So
129	n/a	walk back down the child chain and set the name to be global
130	n/a	rather than free.
131	n/a
132	n/a	Be careful to stop if a child does not think the name is
133	n/a	free.
134	n/a	"""
135	322	self.globals[name] = 1
136	322	if name in self.frees:
137	111	del self.frees[name]
138	676	for child in self.children:
139	354	if child.check_name(name) == SC_FREE:
140	140	child.force_global(name)
141	n/a
142	2	def add_frees(self, names):
143	n/a	"""Process list of free vars from nested scope.
144	n/a
145	n/a	Returns a list of names that are either 1) declared global in the
146	n/a	parent or 2) undefined in a top-level parent. In either case,
147	n/a	the nested scope should treat them as globals.
148	n/a	"""
149	954	child_globals = []
150	1433	for name in names:
151	479	sc = self.check_name(name)
152	479	if self.nested:
153	201	if sc == SC_UNKNOWN or sc == SC_FREE \
154	1	or isinstance(self, ClassScope):
155	200	self.frees[name] = 1
156	1	elif sc == SC_GLOBAL_IMPLICIT:
157	0	child_globals.append(name)
158	1	elif isinstance(self, FunctionScope) and sc == SC_LOCAL:
159	1	self.cells[name] = 1
160	0	elif sc != SC_CELL:
161	0	child_globals.append(name)
162	n/a	else:
163	278	if sc == SC_LOCAL:
164	83	self.cells[name] = 1
165	195	elif sc != SC_CELL:
166	182	child_globals.append(name)
167	954	return child_globals
168	n/a
169	2	def get_cell_vars(self):
170	1028	return self.cells.keys()
171	n/a
172	4	class ModuleScope(Scope):
173	2	__super_init = Scope.__init__
174	n/a
175	2	def __init__(self):
176	31	self.__super_init("global", self)
177	n/a
178	4	class FunctionScope(Scope):
179	2	pass
180	n/a
181	4	class GenExprScope(Scope):
182	2	__super_init = Scope.__init__
183	n/a
184	2	__counter = 1
185	n/a
186	2	def __init__(self, module, klass=None):
187	2	i = self.__counter
188	2	self.__counter += 1
189	2	self.__super_init("generator expression<%d>"%i, module, klass)
190	2	self.add_param('.0')
191	n/a
192	2	def get_names(self):
193	0	keys = Scope.get_names(self)
194	0	return keys
195	n/a
196	4	class LambdaScope(FunctionScope):
197	2	__super_init = Scope.__init__
198	n/a
199	2	__counter = 1
200	n/a
201	2	def __init__(self, module, klass=None):
202	18	i = self.__counter
203	18	self.__counter += 1
204	18	self.__super_init("lambda.%d" % i, module, klass)
205	n/a
206	4	class ClassScope(Scope):
207	2	__super_init = Scope.__init__
208	n/a
209	2	def __init__(self, name, module):
210	391	self.__super_init(name, module, name)
211	n/a
212	4	class SymbolVisitor:
213	2	def __init__(self):
214	31	self.scopes = {}
215	31	self.klass = None
216	n/a
217	n/a	# node that define new scopes
218	n/a
219	2	def visitModule(self, node):
220	31	scope = self.module = self.scopes[node] = ModuleScope()
221	31	self.visit(node.node, scope)
222	n/a
223	2	visitExpression = visitModule
224	n/a
225	2	def visitFunction(self, node, parent):
226	617	if node.decorators:
227	19	self.visit(node.decorators, parent)
228	617	parent.add_def(node.name)
229	660	for n in node.defaults:
230	43	self.visit(n, parent)
231	617	scope = FunctionScope(node.name, self.module, self.klass)
232	617	if parent.nested or isinstance(parent, FunctionScope):
233	306	scope.nested = 1
234	617	self.scopes[node] = scope
235	617	self._do_args(scope, node.argnames)
236	617	self.visit(node.code, scope)
237	617	self.handle_free_vars(scope, parent)
238	n/a
239	2	def visitGenExpr(self, node, parent):
240	2	scope = GenExprScope(self.module, self.klass);
241	2	if parent.nested or isinstance(parent, FunctionScope) \
242	1	or isinstance(parent, GenExprScope):
243	1	scope.nested = 1
244	n/a
245	2	self.scopes[node] = scope
246	2	self.visit(node.code, scope)
247	n/a
248	2	self.handle_free_vars(scope, parent)
249	n/a
250	2	def visitGenExprInner(self, node, scope):
251	5	for genfor in node.quals:
252	3	self.visit(genfor, scope)
253	n/a
254	2	self.visit(node.expr, scope)
255	n/a
256	2	def visitGenExprFor(self, node, scope):
257	3	self.visit(node.assign, scope, 1)
258	3	self.visit(node.iter, scope)
259	5	for if_ in node.ifs:
260	2	self.visit(if_, scope)
261	n/a
262	2	def visitGenExprIf(self, node, scope):
263	2	self.visit(node.test, scope)
264	n/a
265	2	def visitLambda(self, node, parent, assign=0):
266	n/a	# Lambda is an expression, so it could appear in an expression
267	n/a	# context where assign is passed. The transformer should catch
268	n/a	# any code that has a lambda on the left-hand side.
269	18	assert not assign
270	n/a
271	18	for n in node.defaults:
272	0	self.visit(n, parent)
273	18	scope = LambdaScope(self.module, self.klass)
274	18	if parent.nested or isinstance(parent, FunctionScope):
275	18	scope.nested = 1
276	18	self.scopes[node] = scope
277	18	self._do_args(scope, node.argnames)
278	18	self.visit(node.code, scope)
279	18	self.handle_free_vars(scope, parent)
280	n/a
281	2	def _do_args(self, scope, args):
282	1578	for name in args:
283	943	if type(name) == types.TupleType:
284	0	self._do_args(scope, name)
285	n/a	else:
286	943	scope.add_param(name)
287	n/a
288	2	def handle_free_vars(self, scope, parent):
289	1028	parent.add_child(scope)
290	1028	scope.handle_children()
291	n/a
292	2	def visitClass(self, node, parent):
293	391	parent.add_def(node.name)
294	805	for n in node.bases:
295	414	self.visit(n, parent)
296	391	scope = ClassScope(node.name, self.module)
297	391	if parent.nested or isinstance(parent, FunctionScope):
298	368	scope.nested = 1
299	391	if node.doc is not None:
300	7	scope.add_def('__doc__')
301	391	scope.add_def('__module__')
302	391	self.scopes[node] = scope
303	391	prev = self.klass
304	391	self.klass = node.name
305	391	self.visit(node.code, scope)
306	391	self.klass = prev
307	391	self.handle_free_vars(scope, parent)
308	n/a
309	n/a	# name can be a def or a use
310	n/a
311	n/a	# XXX a few calls and nodes expect a third "assign" arg that is
312	n/a	# true if the name is being used as an assignment. only
313	n/a	# expressions contained within statements may have the assign arg.
314	n/a
315	2	def visitName(self, node, scope, assign=0):
316	6826	if assign:
317	11	scope.add_def(node.name)
318	n/a	else:
319	6815	scope.add_use(node.name)
320	n/a
321	n/a	# operations that bind new names
322	n/a
323	2	def visitFor(self, node, scope):
324	90	self.visit(node.assign, scope, 1)
325	90	self.visit(node.list, scope)
326	90	self.visit(node.body, scope)
327	90	if node.else_:
328	0	self.visit(node.else_, scope)
329	n/a
330	2	def visitFrom(self, node, scope):
331	32	for name, asname in node.names:
332	17	if name == "*":
333	0	continue
334	17	scope.add_def(asname or name)
335	n/a
336	2	def visitImport(self, node, scope):
337	101	for name, asname in node.names:
338	53	i = name.find(".")
339	53	if i > -1:
340	2	name = name[:i]
341	53	scope.add_def(asname or name)
342	n/a
343	2	def visitGlobal(self, node, scope):
344	18	for name in node.names:
345	11	scope.add_global(name)
346	n/a
347	2	def visitAssign(self, node, scope):
348	n/a	"""Propagate assignment flag down to child nodes.
349	n/a
350	n/a	The Assign node doesn't itself contains the variables being
351	n/a	assigned to. Instead, the children in node.nodes are visited
352	n/a	with the assign flag set to true. When the names occur in
353	n/a	those nodes, they are marked as defs.
354	n/a
355	n/a	Some names that occur in an assignment target are not bound by
356	n/a	the assignment, e.g. a name occurring inside a slice. The
357	n/a	visitor handles these nodes specially; they do not propagate
358	n/a	the assign flag to their children.
359	n/a	"""
360	1946	for n in node.nodes:
361	975	self.visit(n, scope, 1)
362	971	self.visit(node.expr, scope)
363	n/a
364	2	def visitAssName(self, node, scope, assign=1):
365	829	scope.add_def(node.name)
366	n/a
367	2	def visitAssAttr(self, node, scope, assign=0):
368	337	self.visit(node.expr, scope, 0)
369	n/a
370	2	def visitSubscript(self, node, scope, assign=0):
371	148	self.visit(node.expr, scope, 0)
372	296	for n in node.subs:
373	148	self.visit(n, scope, 0)
374	n/a
375	2	def visitSlice(self, node, scope, assign=0):
376	34	self.visit(node.expr, scope, 0)
377	34	if node.lower:
378	18	self.visit(node.lower, scope, 0)
379	34	if node.upper:
380	20	self.visit(node.upper, scope, 0)
381	n/a
382	2	def visitAugAssign(self, node, scope):
383	n/a	# If the LHS is a name, then this counts as assignment.
384	n/a	# Otherwise, it's just use.
385	17	self.visit(node.node, scope)
386	17	if isinstance(node.node, ast.Name):
387	11	self.visit(node.node, scope, 1) # XXX worry about this
388	17	self.visit(node.expr, scope)
389	n/a
390	n/a	# prune if statements if tests are false
391	n/a
392	2	_const_types = types.StringType, types.IntType, types.FloatType
393	n/a
394	2	def visitIf(self, node, scope):
395	234	for test, body in node.tests:
396	118	if isinstance(test, ast.Const):
397	0	if type(test.value) in self._const_types:
398	0	if not test.value:
399	0	continue
400	118	self.visit(test, scope)
401	118	self.visit(body, scope)
402	116	if node.else_:
403	26	self.visit(node.else_, scope)
404	n/a
405	n/a	# a yield statement signals a generator
406	n/a
407	2	def visitYield(self, node, scope):
408	2	scope.generator = 1
409	2	self.visit(node.value, scope)
410	n/a
411	2	def list_eq(l1, l2):
412	0	return sorted(l1) == sorted(l2)
413	n/a
414	2	if __name__ == "__main__":
415	0	import sys
416	0	from compiler import parseFile, walk
417	0	import symtable
418	n/a
419	0	def get_names(syms):
420	0	return [s for s in [s.get_name() for s in syms.get_symbols()]
421	0	if not (s.startswith('_[') or s.startswith('.'))]
422	n/a
423	0	for file in sys.argv[1:]:
424	0	print file
425	0	f = open(file)
426	0	buf = f.read()
427	0	f.close()
428	0	syms = symtable.symtable(buf, file, "exec")
429	0	mod_names = get_names(syms)
430	0	tree = parseFile(file)
431	0	s = SymbolVisitor()
432	0	walk(tree, s)
433	n/a
434	n/a	# compare module-level symbols
435	0	names2 = s.scopes[tree].get_names()
436	n/a
437	0	if not list_eq(mod_names, names2):
438	0	print
439	0	print "oops", file
440	0	print sorted(mod_names)
441	0	print sorted(names2)
442	0	sys.exit(-1)
443	n/a
444	0	d = {}
445	0	d.update(s.scopes)
446	0	del d[tree]
447	0	scopes = d.values()
448	0	del d
449	n/a
450	0	for s in syms.get_symbols():
451	0	if s.is_namespace():
452	0	l = [sc for sc in scopes
453	0	if sc.name == s.get_name()]
454	0	if len(l) > 1:
455	0	print "skipping", s.get_name()
456	n/a	else:
457	0	if not list_eq(get_names(s.get_namespace()),
458	0	l[0].get_names()):
459	0	print s.get_name()
460	0	print sorted(get_names(s.get_namespace()))
461	0	print sorted(l[0].get_names())
462	0	sys.exit(-1)