ยปCore Development>Code coverage>Lib/lib2to3/patcomp.py

Python code coverage for Lib/lib2to3/patcomp.py

#countcontent
1n/a# Copyright 2006 Google, Inc. All Rights Reserved.
2n/a# Licensed to PSF under a Contributor Agreement.
3n/a
4n/a"""Pattern compiler.
5n/a
6n/aThe grammar is taken from PatternGrammar.txt.
7n/a
8n/aThe compiler compiles a pattern to a pytree.*Pattern instance.
9n/a"""
10n/a
11n/a__author__ = "Guido van Rossum <guido@python.org>"
12n/a
13n/a# Python imports
14n/aimport io
15n/aimport os
16n/a
17n/a# Fairly local imports
18n/afrom .pgen2 import driver, literals, token, tokenize, parse, grammar
19n/a
20n/a# Really local imports
21n/afrom . import pytree
22n/afrom . import pygram
23n/a
24n/a# The pattern grammar file
25n/a_PATTERN_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__),
26n/a "PatternGrammar.txt")
27n/a
28n/a
29n/aclass PatternSyntaxError(Exception):
30n/a pass
31n/a
32n/a
33n/adef tokenize_wrapper(input):
34n/a """Tokenizes a string suppressing significant whitespace."""
35n/a skip = {token.NEWLINE, token.INDENT, token.DEDENT}
36n/a tokens = tokenize.generate_tokens(io.StringIO(input).readline)
37n/a for quintuple in tokens:
38n/a type, value, start, end, line_text = quintuple
39n/a if type not in skip:
40n/a yield quintuple
41n/a
42n/a
43n/aclass PatternCompiler(object):
44n/a
45n/a def __init__(self, grammar_file=_PATTERN_GRAMMAR_FILE):
46n/a """Initializer.
47n/a
48n/a Takes an optional alternative filename for the pattern grammar.
49n/a """
50n/a self.grammar = driver.load_grammar(grammar_file)
51n/a self.syms = pygram.Symbols(self.grammar)
52n/a self.pygrammar = pygram.python_grammar
53n/a self.pysyms = pygram.python_symbols
54n/a self.driver = driver.Driver(self.grammar, convert=pattern_convert)
55n/a
56n/a def compile_pattern(self, input, debug=False, with_tree=False):
57n/a """Compiles a pattern string to a nested pytree.*Pattern object."""
58n/a tokens = tokenize_wrapper(input)
59n/a try:
60n/a root = self.driver.parse_tokens(tokens, debug=debug)
61n/a except parse.ParseError as e:
62n/a raise PatternSyntaxError(str(e))
63n/a if with_tree:
64n/a return self.compile_node(root), root
65n/a else:
66n/a return self.compile_node(root)
67n/a
68n/a def compile_node(self, node):
69n/a """Compiles a node, recursively.
70n/a
71n/a This is one big switch on the node type.
72n/a """
73n/a # XXX Optimize certain Wildcard-containing-Wildcard patterns
74n/a # that can be merged
75n/a if node.type == self.syms.Matcher:
76n/a node = node.children[0] # Avoid unneeded recursion
77n/a
78n/a if node.type == self.syms.Alternatives:
79n/a # Skip the odd children since they are just '|' tokens
80n/a alts = [self.compile_node(ch) for ch in node.children[::2]]
81n/a if len(alts) == 1:
82n/a return alts[0]
83n/a p = pytree.WildcardPattern([[a] for a in alts], min=1, max=1)
84n/a return p.optimize()
85n/a
86n/a if node.type == self.syms.Alternative:
87n/a units = [self.compile_node(ch) for ch in node.children]
88n/a if len(units) == 1:
89n/a return units[0]
90n/a p = pytree.WildcardPattern([units], min=1, max=1)
91n/a return p.optimize()
92n/a
93n/a if node.type == self.syms.NegatedUnit:
94n/a pattern = self.compile_basic(node.children[1:])
95n/a p = pytree.NegatedPattern(pattern)
96n/a return p.optimize()
97n/a
98n/a assert node.type == self.syms.Unit
99n/a
100n/a name = None
101n/a nodes = node.children
102n/a if len(nodes) >= 3 and nodes[1].type == token.EQUAL:
103n/a name = nodes[0].value
104n/a nodes = nodes[2:]
105n/a repeat = None
106n/a if len(nodes) >= 2 and nodes[-1].type == self.syms.Repeater:
107n/a repeat = nodes[-1]
108n/a nodes = nodes[:-1]
109n/a
110n/a # Now we've reduced it to: STRING | NAME [Details] | (...) | [...]
111n/a pattern = self.compile_basic(nodes, repeat)
112n/a
113n/a if repeat is not None:
114n/a assert repeat.type == self.syms.Repeater
115n/a children = repeat.children
116n/a child = children[0]
117n/a if child.type == token.STAR:
118n/a min = 0
119n/a max = pytree.HUGE
120n/a elif child.type == token.PLUS:
121n/a min = 1
122n/a max = pytree.HUGE
123n/a elif child.type == token.LBRACE:
124n/a assert children[-1].type == token.RBRACE
125n/a assert len(children) in (3, 5)
126n/a min = max = self.get_int(children[1])
127n/a if len(children) == 5:
128n/a max = self.get_int(children[3])
129n/a else:
130n/a assert False
131n/a if min != 1 or max != 1:
132n/a pattern = pattern.optimize()
133n/a pattern = pytree.WildcardPattern([[pattern]], min=min, max=max)
134n/a
135n/a if name is not None:
136n/a pattern.name = name
137n/a return pattern.optimize()
138n/a
139n/a def compile_basic(self, nodes, repeat=None):
140n/a # Compile STRING | NAME [Details] | (...) | [...]
141n/a assert len(nodes) >= 1
142n/a node = nodes[0]
143n/a if node.type == token.STRING:
144n/a value = str(literals.evalString(node.value))
145n/a return pytree.LeafPattern(_type_of_literal(value), value)
146n/a elif node.type == token.NAME:
147n/a value = node.value
148n/a if value.isupper():
149n/a if value not in TOKEN_MAP:
150n/a raise PatternSyntaxError("Invalid token: %r" % value)
151n/a if nodes[1:]:
152n/a raise PatternSyntaxError("Can't have details for token")
153n/a return pytree.LeafPattern(TOKEN_MAP[value])
154n/a else:
155n/a if value == "any":
156n/a type = None
157n/a elif not value.startswith("_"):
158n/a type = getattr(self.pysyms, value, None)
159n/a if type is None:
160n/a raise PatternSyntaxError("Invalid symbol: %r" % value)
161n/a if nodes[1:]: # Details present
162n/a content = [self.compile_node(nodes[1].children[1])]
163n/a else:
164n/a content = None
165n/a return pytree.NodePattern(type, content)
166n/a elif node.value == "(":
167n/a return self.compile_node(nodes[1])
168n/a elif node.value == "[":
169n/a assert repeat is None
170n/a subpattern = self.compile_node(nodes[1])
171n/a return pytree.WildcardPattern([[subpattern]], min=0, max=1)
172n/a assert False, node
173n/a
174n/a def get_int(self, node):
175n/a assert node.type == token.NUMBER
176n/a return int(node.value)
177n/a
178n/a
179n/a# Map named tokens to the type value for a LeafPattern
180n/aTOKEN_MAP = {"NAME": token.NAME,
181n/a "STRING": token.STRING,
182n/a "NUMBER": token.NUMBER,
183n/a "TOKEN": None}
184n/a
185n/a
186n/adef _type_of_literal(value):
187n/a if value[0].isalpha():
188n/a return token.NAME
189n/a elif value in grammar.opmap:
190n/a return grammar.opmap[value]
191n/a else:
192n/a return None
193n/a
194n/a
195n/adef pattern_convert(grammar, raw_node_info):
196n/a """Converts raw node information to a Node or Leaf instance."""
197n/a type, value, context, children = raw_node_info
198n/a if children or type in grammar.number2symbol:
199n/a return pytree.Node(type, children, context=context)
200n/a else:
201n/a return pytree.Leaf(type, value, context=context)
202n/a
203n/a
204n/adef compile_pattern(pattern):
205n/a return PatternCompiler().compile_pattern(pattern)