Python code coverage for Lib/lib2to3/patcomp.py

#	count	content
1	n/a	# Copyright 2006 Google, Inc. All Rights Reserved.
2	n/a	# Licensed to PSF under a Contributor Agreement.
3	n/a
4	n/a	"""Pattern compiler.
5	n/a
6	n/a	The grammar is taken from PatternGrammar.txt.
7	n/a
8	n/a	The compiler compiles a pattern to a pytree.*Pattern instance.
9	n/a	"""
10	n/a
11	n/a	__author__ = "Guido van Rossum <guido@python.org>"
12	n/a
13	n/a	# Python imports
14	n/a	import io
15	n/a	import os
16	n/a
17	n/a	# Fairly local imports
18	n/a	from .pgen2 import driver, literals, token, tokenize, parse, grammar
19	n/a
20	n/a	# Really local imports
21	n/a	from . import pytree
22	n/a	from . import pygram
23	n/a
24	n/a	# The pattern grammar file
25	n/a	_PATTERN_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__),
26	n/a	"PatternGrammar.txt")
27	n/a
28	n/a
29	n/a	class PatternSyntaxError(Exception):
30	n/a	pass
31	n/a
32	n/a
33	n/a	def tokenize_wrapper(input):
34	n/a	"""Tokenizes a string suppressing significant whitespace."""
35	n/a	skip = {token.NEWLINE, token.INDENT, token.DEDENT}
36	n/a	tokens = tokenize.generate_tokens(io.StringIO(input).readline)
37	n/a	for quintuple in tokens:
38	n/a	type, value, start, end, line_text = quintuple
39	n/a	if type not in skip:
40	n/a	yield quintuple
41	n/a
42	n/a
43	n/a	class PatternCompiler(object):
44	n/a
45	n/a	def __init__(self, grammar_file=_PATTERN_GRAMMAR_FILE):
46	n/a	"""Initializer.
47	n/a
48	n/a	Takes an optional alternative filename for the pattern grammar.
49	n/a	"""
50	n/a	self.grammar = driver.load_grammar(grammar_file)
51	n/a	self.syms = pygram.Symbols(self.grammar)
52	n/a	self.pygrammar = pygram.python_grammar
53	n/a	self.pysyms = pygram.python_symbols
54	n/a	self.driver = driver.Driver(self.grammar, convert=pattern_convert)
55	n/a
56	n/a	def compile_pattern(self, input, debug=False, with_tree=False):
57	n/a	"""Compiles a pattern string to a nested pytree.*Pattern object."""
58	n/a	tokens = tokenize_wrapper(input)
59	n/a	try:
60	n/a	root = self.driver.parse_tokens(tokens, debug=debug)
61	n/a	except parse.ParseError as e:
62	n/a	raise PatternSyntaxError(str(e))
63	n/a	if with_tree:
64	n/a	return self.compile_node(root), root
65	n/a	else:
66	n/a	return self.compile_node(root)
67	n/a
68	n/a	def compile_node(self, node):
69	n/a	"""Compiles a node, recursively.
70	n/a
71	n/a	This is one big switch on the node type.
72	n/a	"""
73	n/a	# XXX Optimize certain Wildcard-containing-Wildcard patterns
74	n/a	# that can be merged
75	n/a	if node.type == self.syms.Matcher:
76	n/a	node = node.children[0] # Avoid unneeded recursion
77	n/a
78	n/a	if node.type == self.syms.Alternatives:
79	n/a	# Skip the odd children since they are just '\|' tokens
80	n/a	alts = [self.compile_node(ch) for ch in node.children[::2]]
81	n/a	if len(alts) == 1:
82	n/a	return alts[0]
83	n/a	p = pytree.WildcardPattern([[a] for a in alts], min=1, max=1)
84	n/a	return p.optimize()
85	n/a
86	n/a	if node.type == self.syms.Alternative:
87	n/a	units = [self.compile_node(ch) for ch in node.children]
88	n/a	if len(units) == 1:
89	n/a	return units[0]
90	n/a	p = pytree.WildcardPattern([units], min=1, max=1)
91	n/a	return p.optimize()
92	n/a
93	n/a	if node.type == self.syms.NegatedUnit:
94	n/a	pattern = self.compile_basic(node.children[1:])
95	n/a	p = pytree.NegatedPattern(pattern)
96	n/a	return p.optimize()
97	n/a
98	n/a	assert node.type == self.syms.Unit
99	n/a
100	n/a	name = None
101	n/a	nodes = node.children
102	n/a	if len(nodes) >= 3 and nodes[1].type == token.EQUAL:
103	n/a	name = nodes[0].value
104	n/a	nodes = nodes[2:]
105	n/a	repeat = None
106	n/a	if len(nodes) >= 2 and nodes[-1].type == self.syms.Repeater:
107	n/a	repeat = nodes[-1]
108	n/a	nodes = nodes[:-1]
109	n/a
110	n/a	# Now we've reduced it to: STRING \| NAME [Details] \| (...) \| [...]
111	n/a	pattern = self.compile_basic(nodes, repeat)
112	n/a
113	n/a	if repeat is not None:
114	n/a	assert repeat.type == self.syms.Repeater
115	n/a	children = repeat.children
116	n/a	child = children[0]
117	n/a	if child.type == token.STAR:
118	n/a	min = 0
119	n/a	max = pytree.HUGE
120	n/a	elif child.type == token.PLUS:
121	n/a	min = 1
122	n/a	max = pytree.HUGE
123	n/a	elif child.type == token.LBRACE:
124	n/a	assert children[-1].type == token.RBRACE
125	n/a	assert len(children) in (3, 5)
126	n/a	min = max = self.get_int(children[1])
127	n/a	if len(children) == 5:
128	n/a	max = self.get_int(children[3])
129	n/a	else:
130	n/a	assert False
131	n/a	if min != 1 or max != 1:
132	n/a	pattern = pattern.optimize()
133	n/a	pattern = pytree.WildcardPattern([[pattern]], min=min, max=max)
134	n/a
135	n/a	if name is not None:
136	n/a	pattern.name = name
137	n/a	return pattern.optimize()
138	n/a
139	n/a	def compile_basic(self, nodes, repeat=None):
140	n/a	# Compile STRING \| NAME [Details] \| (...) \| [...]
141	n/a	assert len(nodes) >= 1
142	n/a	node = nodes[0]
143	n/a	if node.type == token.STRING:
144	n/a	value = str(literals.evalString(node.value))
145	n/a	return pytree.LeafPattern(_type_of_literal(value), value)
146	n/a	elif node.type == token.NAME:
147	n/a	value = node.value
148	n/a	if value.isupper():
149	n/a	if value not in TOKEN_MAP:
150	n/a	raise PatternSyntaxError("Invalid token: %r" % value)
151	n/a	if nodes[1:]:
152	n/a	raise PatternSyntaxError("Can't have details for token")
153	n/a	return pytree.LeafPattern(TOKEN_MAP[value])
154	n/a	else:
155	n/a	if value == "any":
156	n/a	type = None
157	n/a	elif not value.startswith("_"):
158	n/a	type = getattr(self.pysyms, value, None)
159	n/a	if type is None:
160	n/a	raise PatternSyntaxError("Invalid symbol: %r" % value)
161	n/a	if nodes[1:]: # Details present
162	n/a	content = [self.compile_node(nodes[1].children[1])]
163	n/a	else:
164	n/a	content = None
165	n/a	return pytree.NodePattern(type, content)
166	n/a	elif node.value == "(":
167	n/a	return self.compile_node(nodes[1])
168	n/a	elif node.value == "[":
169	n/a	assert repeat is None
170	n/a	subpattern = self.compile_node(nodes[1])
171	n/a	return pytree.WildcardPattern([[subpattern]], min=0, max=1)
172	n/a	assert False, node
173	n/a
174	n/a	def get_int(self, node):
175	n/a	assert node.type == token.NUMBER
176	n/a	return int(node.value)
177	n/a
178	n/a
179	n/a	# Map named tokens to the type value for a LeafPattern
180	n/a	TOKEN_MAP = {"NAME": token.NAME,
181	n/a	"STRING": token.STRING,
182	n/a	"NUMBER": token.NUMBER,
183	n/a	"TOKEN": None}
184	n/a
185	n/a
186	n/a	def _type_of_literal(value):
187	n/a	if value[0].isalpha():
188	n/a	return token.NAME
189	n/a	elif value in grammar.opmap:
190	n/a	return grammar.opmap[value]
191	n/a	else:
192	n/a	return None
193	n/a
194	n/a
195	n/a	def pattern_convert(grammar, raw_node_info):
196	n/a	"""Converts raw node information to a Node or Leaf instance."""
197	n/a	type, value, context, children = raw_node_info
198	n/a	if children or type in grammar.number2symbol:
199	n/a	return pytree.Node(type, children, context=context)
200	n/a	else:
201	n/a	return pytree.Leaf(type, value, context=context)
202	n/a
203	n/a
204	n/a	def compile_pattern(pattern):
205	n/a	return PatternCompiler().compile_pattern(pattern)