ยปCore Development>Code coverage>Tools/parser/unparse.py

Python code coverage for Tools/parser/unparse.py

#countcontent
1n/a"Usage: unparse.py <path to source file>"
2n/aimport sys
3n/aimport ast
4n/aimport tokenize
5n/aimport io
6n/aimport os
7n/a
8n/a# Large float and imaginary literals get turned into infinities in the AST.
9n/a# We unparse those infinities to INFSTR.
10n/aINFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)
11n/a
12n/adef interleave(inter, f, seq):
13n/a """Call f on each item in seq, calling inter() in between.
14n/a """
15n/a seq = iter(seq)
16n/a try:
17n/a f(next(seq))
18n/a except StopIteration:
19n/a pass
20n/a else:
21n/a for x in seq:
22n/a inter()
23n/a f(x)
24n/a
25n/aclass Unparser:
26n/a """Methods in this class recursively traverse an AST and
27n/a output source code for the abstract syntax; original formatting
28n/a is disregarded. """
29n/a
30n/a def __init__(self, tree, file = sys.stdout):
31n/a """Unparser(tree, file=sys.stdout) -> None.
32n/a Print the source for tree to file."""
33n/a self.f = file
34n/a self._indent = 0
35n/a self.dispatch(tree)
36n/a print("", file=self.f)
37n/a self.f.flush()
38n/a
39n/a def fill(self, text = ""):
40n/a "Indent a piece of text, according to the current indentation level"
41n/a self.f.write("\n"+" "*self._indent + text)
42n/a
43n/a def write(self, text):
44n/a "Append a piece of text to the current line."
45n/a self.f.write(text)
46n/a
47n/a def enter(self):
48n/a "Print ':', and increase the indentation."
49n/a self.write(":")
50n/a self._indent += 1
51n/a
52n/a def leave(self):
53n/a "Decrease the indentation level."
54n/a self._indent -= 1
55n/a
56n/a def dispatch(self, tree):
57n/a "Dispatcher function, dispatching tree type T to method _T."
58n/a if isinstance(tree, list):
59n/a for t in tree:
60n/a self.dispatch(t)
61n/a return
62n/a meth = getattr(self, "_"+tree.__class__.__name__)
63n/a meth(tree)
64n/a
65n/a
66n/a ############### Unparsing methods ######################
67n/a # There should be one method per concrete grammar type #
68n/a # Constructors should be grouped by sum type. Ideally, #
69n/a # this would follow the order in the grammar, but #
70n/a # currently doesn't. #
71n/a ########################################################
72n/a
73n/a def _Module(self, tree):
74n/a for stmt in tree.body:
75n/a self.dispatch(stmt)
76n/a
77n/a # stmt
78n/a def _Expr(self, tree):
79n/a self.fill()
80n/a self.dispatch(tree.value)
81n/a
82n/a def _Import(self, t):
83n/a self.fill("import ")
84n/a interleave(lambda: self.write(", "), self.dispatch, t.names)
85n/a
86n/a def _ImportFrom(self, t):
87n/a self.fill("from ")
88n/a self.write("." * t.level)
89n/a if t.module:
90n/a self.write(t.module)
91n/a self.write(" import ")
92n/a interleave(lambda: self.write(", "), self.dispatch, t.names)
93n/a
94n/a def _Assign(self, t):
95n/a self.fill()
96n/a for target in t.targets:
97n/a self.dispatch(target)
98n/a self.write(" = ")
99n/a self.dispatch(t.value)
100n/a
101n/a def _AugAssign(self, t):
102n/a self.fill()
103n/a self.dispatch(t.target)
104n/a self.write(" "+self.binop[t.op.__class__.__name__]+"= ")
105n/a self.dispatch(t.value)
106n/a
107n/a def _AnnAssign(self, t):
108n/a self.fill()
109n/a if not t.simple and isinstance(t.target, ast.Name):
110n/a self.write('(')
111n/a self.dispatch(t.target)
112n/a if not t.simple and isinstance(t.target, ast.Name):
113n/a self.write(')')
114n/a self.write(": ")
115n/a self.dispatch(t.annotation)
116n/a if t.value:
117n/a self.write(" = ")
118n/a self.dispatch(t.value)
119n/a
120n/a def _Return(self, t):
121n/a self.fill("return")
122n/a if t.value:
123n/a self.write(" ")
124n/a self.dispatch(t.value)
125n/a
126n/a def _Pass(self, t):
127n/a self.fill("pass")
128n/a
129n/a def _Break(self, t):
130n/a self.fill("break")
131n/a
132n/a def _Continue(self, t):
133n/a self.fill("continue")
134n/a
135n/a def _Delete(self, t):
136n/a self.fill("del ")
137n/a interleave(lambda: self.write(", "), self.dispatch, t.targets)
138n/a
139n/a def _Assert(self, t):
140n/a self.fill("assert ")
141n/a self.dispatch(t.test)
142n/a if t.msg:
143n/a self.write(", ")
144n/a self.dispatch(t.msg)
145n/a
146n/a def _Global(self, t):
147n/a self.fill("global ")
148n/a interleave(lambda: self.write(", "), self.write, t.names)
149n/a
150n/a def _Nonlocal(self, t):
151n/a self.fill("nonlocal ")
152n/a interleave(lambda: self.write(", "), self.write, t.names)
153n/a
154n/a def _Await(self, t):
155n/a self.write("(")
156n/a self.write("await")
157n/a if t.value:
158n/a self.write(" ")
159n/a self.dispatch(t.value)
160n/a self.write(")")
161n/a
162n/a def _Yield(self, t):
163n/a self.write("(")
164n/a self.write("yield")
165n/a if t.value:
166n/a self.write(" ")
167n/a self.dispatch(t.value)
168n/a self.write(")")
169n/a
170n/a def _YieldFrom(self, t):
171n/a self.write("(")
172n/a self.write("yield from")
173n/a if t.value:
174n/a self.write(" ")
175n/a self.dispatch(t.value)
176n/a self.write(")")
177n/a
178n/a def _Raise(self, t):
179n/a self.fill("raise")
180n/a if not t.exc:
181n/a assert not t.cause
182n/a return
183n/a self.write(" ")
184n/a self.dispatch(t.exc)
185n/a if t.cause:
186n/a self.write(" from ")
187n/a self.dispatch(t.cause)
188n/a
189n/a def _Try(self, t):
190n/a self.fill("try")
191n/a self.enter()
192n/a self.dispatch(t.body)
193n/a self.leave()
194n/a for ex in t.handlers:
195n/a self.dispatch(ex)
196n/a if t.orelse:
197n/a self.fill("else")
198n/a self.enter()
199n/a self.dispatch(t.orelse)
200n/a self.leave()
201n/a if t.finalbody:
202n/a self.fill("finally")
203n/a self.enter()
204n/a self.dispatch(t.finalbody)
205n/a self.leave()
206n/a
207n/a def _ExceptHandler(self, t):
208n/a self.fill("except")
209n/a if t.type:
210n/a self.write(" ")
211n/a self.dispatch(t.type)
212n/a if t.name:
213n/a self.write(" as ")
214n/a self.write(t.name)
215n/a self.enter()
216n/a self.dispatch(t.body)
217n/a self.leave()
218n/a
219n/a def _ClassDef(self, t):
220n/a self.write("\n")
221n/a for deco in t.decorator_list:
222n/a self.fill("@")
223n/a self.dispatch(deco)
224n/a self.fill("class "+t.name)
225n/a self.write("(")
226n/a comma = False
227n/a for e in t.bases:
228n/a if comma: self.write(", ")
229n/a else: comma = True
230n/a self.dispatch(e)
231n/a for e in t.keywords:
232n/a if comma: self.write(", ")
233n/a else: comma = True
234n/a self.dispatch(e)
235n/a self.write(")")
236n/a
237n/a self.enter()
238n/a self.dispatch(t.body)
239n/a self.leave()
240n/a
241n/a def _FunctionDef(self, t):
242n/a self.__FunctionDef_helper(t, "def")
243n/a
244n/a def _AsyncFunctionDef(self, t):
245n/a self.__FunctionDef_helper(t, "async def")
246n/a
247n/a def __FunctionDef_helper(self, t, fill_suffix):
248n/a self.write("\n")
249n/a for deco in t.decorator_list:
250n/a self.fill("@")
251n/a self.dispatch(deco)
252n/a def_str = fill_suffix+" "+t.name + "("
253n/a self.fill(def_str)
254n/a self.dispatch(t.args)
255n/a self.write(")")
256n/a if t.returns:
257n/a self.write(" -> ")
258n/a self.dispatch(t.returns)
259n/a self.enter()
260n/a self.dispatch(t.body)
261n/a self.leave()
262n/a
263n/a def _For(self, t):
264n/a self.__For_helper("for ", t)
265n/a
266n/a def _AsyncFor(self, t):
267n/a self.__For_helper("async for ", t)
268n/a
269n/a def __For_helper(self, fill, t):
270n/a self.fill(fill)
271n/a self.dispatch(t.target)
272n/a self.write(" in ")
273n/a self.dispatch(t.iter)
274n/a self.enter()
275n/a self.dispatch(t.body)
276n/a self.leave()
277n/a if t.orelse:
278n/a self.fill("else")
279n/a self.enter()
280n/a self.dispatch(t.orelse)
281n/a self.leave()
282n/a
283n/a def _If(self, t):
284n/a self.fill("if ")
285n/a self.dispatch(t.test)
286n/a self.enter()
287n/a self.dispatch(t.body)
288n/a self.leave()
289n/a # collapse nested ifs into equivalent elifs.
290n/a while (t.orelse and len(t.orelse) == 1 and
291n/a isinstance(t.orelse[0], ast.If)):
292n/a t = t.orelse[0]
293n/a self.fill("elif ")
294n/a self.dispatch(t.test)
295n/a self.enter()
296n/a self.dispatch(t.body)
297n/a self.leave()
298n/a # final else
299n/a if t.orelse:
300n/a self.fill("else")
301n/a self.enter()
302n/a self.dispatch(t.orelse)
303n/a self.leave()
304n/a
305n/a def _While(self, t):
306n/a self.fill("while ")
307n/a self.dispatch(t.test)
308n/a self.enter()
309n/a self.dispatch(t.body)
310n/a self.leave()
311n/a if t.orelse:
312n/a self.fill("else")
313n/a self.enter()
314n/a self.dispatch(t.orelse)
315n/a self.leave()
316n/a
317n/a def _With(self, t):
318n/a self.fill("with ")
319n/a interleave(lambda: self.write(", "), self.dispatch, t.items)
320n/a self.enter()
321n/a self.dispatch(t.body)
322n/a self.leave()
323n/a
324n/a def _AsyncWith(self, t):
325n/a self.fill("async with ")
326n/a interleave(lambda: self.write(", "), self.dispatch, t.items)
327n/a self.enter()
328n/a self.dispatch(t.body)
329n/a self.leave()
330n/a
331n/a # expr
332n/a def _Bytes(self, t):
333n/a self.write(repr(t.s))
334n/a
335n/a def _Str(self, tree):
336n/a self.write(repr(tree.s))
337n/a
338n/a def _JoinedStr(self, t):
339n/a self.write("f")
340n/a string = io.StringIO()
341n/a self._fstring_JoinedStr(t, string.write)
342n/a self.write(repr(string.getvalue()))
343n/a
344n/a def _FormattedValue(self, t):
345n/a self.write("f")
346n/a string = io.StringIO()
347n/a self._fstring_FormattedValue(t, string.write)
348n/a self.write(repr(string.getvalue()))
349n/a
350n/a def _fstring_JoinedStr(self, t, write):
351n/a for value in t.values:
352n/a meth = getattr(self, "_fstring_" + type(value).__name__)
353n/a meth(value, write)
354n/a
355n/a def _fstring_Str(self, t, write):
356n/a value = t.s.replace("{", "{{").replace("}", "}}")
357n/a write(value)
358n/a
359n/a def _fstring_Constant(self, t, write):
360n/a assert isinstance(t.value, str)
361n/a value = t.value.replace("{", "{{").replace("}", "}}")
362n/a write(value)
363n/a
364n/a def _fstring_FormattedValue(self, t, write):
365n/a write("{")
366n/a expr = io.StringIO()
367n/a Unparser(t.value, expr)
368n/a expr = expr.getvalue().rstrip("\n")
369n/a if expr.startswith("{"):
370n/a write(" ") # Separate pair of opening brackets as "{ {"
371n/a write(expr)
372n/a if t.conversion != -1:
373n/a conversion = chr(t.conversion)
374n/a assert conversion in "sra"
375n/a write(f"!{conversion}")
376n/a if t.format_spec:
377n/a write(":")
378n/a meth = getattr(self, "_fstring_" + type(t.format_spec).__name__)
379n/a meth(t.format_spec, write)
380n/a write("}")
381n/a
382n/a def _Name(self, t):
383n/a self.write(t.id)
384n/a
385n/a def _write_constant(self, value):
386n/a if isinstance(value, (float, complex)):
387n/a self.write(repr(value).replace("inf", INFSTR))
388n/a else:
389n/a self.write(repr(value))
390n/a
391n/a def _Constant(self, t):
392n/a value = t.value
393n/a if isinstance(value, tuple):
394n/a self.write("(")
395n/a if len(value) == 1:
396n/a self._write_constant(value[0])
397n/a self.write(",")
398n/a else:
399n/a interleave(lambda: self.write(", "), self._write_constant, value)
400n/a self.write(")")
401n/a else:
402n/a self._write_constant(t.value)
403n/a
404n/a def _NameConstant(self, t):
405n/a self.write(repr(t.value))
406n/a
407n/a def _Num(self, t):
408n/a # Substitute overflowing decimal literal for AST infinities.
409n/a self.write(repr(t.n).replace("inf", INFSTR))
410n/a
411n/a def _List(self, t):
412n/a self.write("[")
413n/a interleave(lambda: self.write(", "), self.dispatch, t.elts)
414n/a self.write("]")
415n/a
416n/a def _ListComp(self, t):
417n/a self.write("[")
418n/a self.dispatch(t.elt)
419n/a for gen in t.generators:
420n/a self.dispatch(gen)
421n/a self.write("]")
422n/a
423n/a def _GeneratorExp(self, t):
424n/a self.write("(")
425n/a self.dispatch(t.elt)
426n/a for gen in t.generators:
427n/a self.dispatch(gen)
428n/a self.write(")")
429n/a
430n/a def _SetComp(self, t):
431n/a self.write("{")
432n/a self.dispatch(t.elt)
433n/a for gen in t.generators:
434n/a self.dispatch(gen)
435n/a self.write("}")
436n/a
437n/a def _DictComp(self, t):
438n/a self.write("{")
439n/a self.dispatch(t.key)
440n/a self.write(": ")
441n/a self.dispatch(t.value)
442n/a for gen in t.generators:
443n/a self.dispatch(gen)
444n/a self.write("}")
445n/a
446n/a def _comprehension(self, t):
447n/a if t.is_async:
448n/a self.write(" async for ")
449n/a else:
450n/a self.write(" for ")
451n/a self.dispatch(t.target)
452n/a self.write(" in ")
453n/a self.dispatch(t.iter)
454n/a for if_clause in t.ifs:
455n/a self.write(" if ")
456n/a self.dispatch(if_clause)
457n/a
458n/a def _IfExp(self, t):
459n/a self.write("(")
460n/a self.dispatch(t.body)
461n/a self.write(" if ")
462n/a self.dispatch(t.test)
463n/a self.write(" else ")
464n/a self.dispatch(t.orelse)
465n/a self.write(")")
466n/a
467n/a def _Set(self, t):
468n/a assert(t.elts) # should be at least one element
469n/a self.write("{")
470n/a interleave(lambda: self.write(", "), self.dispatch, t.elts)
471n/a self.write("}")
472n/a
473n/a def _Dict(self, t):
474n/a self.write("{")
475n/a def write_key_value_pair(k, v):
476n/a self.dispatch(k)
477n/a self.write(": ")
478n/a self.dispatch(v)
479n/a
480n/a def write_item(item):
481n/a k, v = item
482n/a if k is None:
483n/a # for dictionary unpacking operator in dicts {**{'y': 2}}
484n/a # see PEP 448 for details
485n/a self.write("**")
486n/a self.dispatch(v)
487n/a else:
488n/a write_key_value_pair(k, v)
489n/a interleave(lambda: self.write(", "), write_item, zip(t.keys, t.values))
490n/a self.write("}")
491n/a
492n/a def _Tuple(self, t):
493n/a self.write("(")
494n/a if len(t.elts) == 1:
495n/a elt = t.elts[0]
496n/a self.dispatch(elt)
497n/a self.write(",")
498n/a else:
499n/a interleave(lambda: self.write(", "), self.dispatch, t.elts)
500n/a self.write(")")
501n/a
502n/a unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"}
503n/a def _UnaryOp(self, t):
504n/a self.write("(")
505n/a self.write(self.unop[t.op.__class__.__name__])
506n/a self.write(" ")
507n/a self.dispatch(t.operand)
508n/a self.write(")")
509n/a
510n/a binop = { "Add":"+", "Sub":"-", "Mult":"*", "MatMult":"@", "Div":"/", "Mod":"%",
511n/a "LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&",
512n/a "FloorDiv":"//", "Pow": "**"}
513n/a def _BinOp(self, t):
514n/a self.write("(")
515n/a self.dispatch(t.left)
516n/a self.write(" " + self.binop[t.op.__class__.__name__] + " ")
517n/a self.dispatch(t.right)
518n/a self.write(")")
519n/a
520n/a cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=",
521n/a "Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"}
522n/a def _Compare(self, t):
523n/a self.write("(")
524n/a self.dispatch(t.left)
525n/a for o, e in zip(t.ops, t.comparators):
526n/a self.write(" " + self.cmpops[o.__class__.__name__] + " ")
527n/a self.dispatch(e)
528n/a self.write(")")
529n/a
530n/a boolops = {ast.And: 'and', ast.Or: 'or'}
531n/a def _BoolOp(self, t):
532n/a self.write("(")
533n/a s = " %s " % self.boolops[t.op.__class__]
534n/a interleave(lambda: self.write(s), self.dispatch, t.values)
535n/a self.write(")")
536n/a
537n/a def _Attribute(self,t):
538n/a self.dispatch(t.value)
539n/a # Special case: 3.__abs__() is a syntax error, so if t.value
540n/a # is an integer literal then we need to either parenthesize
541n/a # it or add an extra space to get 3 .__abs__().
542n/a if ((isinstance(t.value, ast.Num) and isinstance(t.value.n, int))
543n/a or (isinstance(t.value, ast.Constant) and isinstance(t.value.value, int))):
544n/a self.write(" ")
545n/a self.write(".")
546n/a self.write(t.attr)
547n/a
548n/a def _Call(self, t):
549n/a self.dispatch(t.func)
550n/a self.write("(")
551n/a comma = False
552n/a for e in t.args:
553n/a if comma: self.write(", ")
554n/a else: comma = True
555n/a self.dispatch(e)
556n/a for e in t.keywords:
557n/a if comma: self.write(", ")
558n/a else: comma = True
559n/a self.dispatch(e)
560n/a self.write(")")
561n/a
562n/a def _Subscript(self, t):
563n/a self.dispatch(t.value)
564n/a self.write("[")
565n/a self.dispatch(t.slice)
566n/a self.write("]")
567n/a
568n/a def _Starred(self, t):
569n/a self.write("*")
570n/a self.dispatch(t.value)
571n/a
572n/a # slice
573n/a def _Ellipsis(self, t):
574n/a self.write("...")
575n/a
576n/a def _Index(self, t):
577n/a self.dispatch(t.value)
578n/a
579n/a def _Slice(self, t):
580n/a if t.lower:
581n/a self.dispatch(t.lower)
582n/a self.write(":")
583n/a if t.upper:
584n/a self.dispatch(t.upper)
585n/a if t.step:
586n/a self.write(":")
587n/a self.dispatch(t.step)
588n/a
589n/a def _ExtSlice(self, t):
590n/a interleave(lambda: self.write(', '), self.dispatch, t.dims)
591n/a
592n/a # argument
593n/a def _arg(self, t):
594n/a self.write(t.arg)
595n/a if t.annotation:
596n/a self.write(": ")
597n/a self.dispatch(t.annotation)
598n/a
599n/a # others
600n/a def _arguments(self, t):
601n/a first = True
602n/a # normal arguments
603n/a defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults
604n/a for a, d in zip(t.args, defaults):
605n/a if first:first = False
606n/a else: self.write(", ")
607n/a self.dispatch(a)
608n/a if d:
609n/a self.write("=")
610n/a self.dispatch(d)
611n/a
612n/a # varargs, or bare '*' if no varargs but keyword-only arguments present
613n/a if t.vararg or t.kwonlyargs:
614n/a if first:first = False
615n/a else: self.write(", ")
616n/a self.write("*")
617n/a if t.vararg:
618n/a self.write(t.vararg.arg)
619n/a if t.vararg.annotation:
620n/a self.write(": ")
621n/a self.dispatch(t.vararg.annotation)
622n/a
623n/a # keyword-only arguments
624n/a if t.kwonlyargs:
625n/a for a, d in zip(t.kwonlyargs, t.kw_defaults):
626n/a if first:first = False
627n/a else: self.write(", ")
628n/a self.dispatch(a),
629n/a if d:
630n/a self.write("=")
631n/a self.dispatch(d)
632n/a
633n/a # kwargs
634n/a if t.kwarg:
635n/a if first:first = False
636n/a else: self.write(", ")
637n/a self.write("**"+t.kwarg.arg)
638n/a if t.kwarg.annotation:
639n/a self.write(": ")
640n/a self.dispatch(t.kwarg.annotation)
641n/a
642n/a def _keyword(self, t):
643n/a if t.arg is None:
644n/a self.write("**")
645n/a else:
646n/a self.write(t.arg)
647n/a self.write("=")
648n/a self.dispatch(t.value)
649n/a
650n/a def _Lambda(self, t):
651n/a self.write("(")
652n/a self.write("lambda ")
653n/a self.dispatch(t.args)
654n/a self.write(": ")
655n/a self.dispatch(t.body)
656n/a self.write(")")
657n/a
658n/a def _alias(self, t):
659n/a self.write(t.name)
660n/a if t.asname:
661n/a self.write(" as "+t.asname)
662n/a
663n/a def _withitem(self, t):
664n/a self.dispatch(t.context_expr)
665n/a if t.optional_vars:
666n/a self.write(" as ")
667n/a self.dispatch(t.optional_vars)
668n/a
669n/adef roundtrip(filename, output=sys.stdout):
670n/a with open(filename, "rb") as pyfile:
671n/a encoding = tokenize.detect_encoding(pyfile.readline)[0]
672n/a with open(filename, "r", encoding=encoding) as pyfile:
673n/a source = pyfile.read()
674n/a tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST)
675n/a Unparser(tree, output)
676n/a
677n/a
678n/a
679n/adef testdir(a):
680n/a try:
681n/a names = [n for n in os.listdir(a) if n.endswith('.py')]
682n/a except OSError:
683n/a print("Directory not readable: %s" % a, file=sys.stderr)
684n/a else:
685n/a for n in names:
686n/a fullname = os.path.join(a, n)
687n/a if os.path.isfile(fullname):
688n/a output = io.StringIO()
689n/a print('Testing %s' % fullname)
690n/a try:
691n/a roundtrip(fullname, output)
692n/a except Exception as e:
693n/a print(' Failed to compile, exception is %s' % repr(e))
694n/a elif os.path.isdir(fullname):
695n/a testdir(fullname)
696n/a
697n/adef main(args):
698n/a if args[0] == '--testdir':
699n/a for a in args[1:]:
700n/a testdir(a)
701n/a else:
702n/a for a in args:
703n/a roundtrip(a)
704n/a
705n/aif __name__=='__main__':
706n/a main(sys.argv[1:])