ยปCore Development>Code coverage>Lib/lib2to3/refactor.py

Python code coverage for Lib/lib2to3/refactor.py

#countcontent
1n/a# Copyright 2006 Google, Inc. All Rights Reserved.
2n/a# Licensed to PSF under a Contributor Agreement.
3n/a
4n/a"""Refactoring framework.
5n/a
6n/aUsed as a main program, this can refactor any number of files and/or
7n/arecursively descend down directories. Imported as a module, this
8n/aprovides infrastructure to write your own refactoring tool.
9n/a"""
10n/a
11n/a__author__ = "Guido van Rossum <guido@python.org>"
12n/a
13n/a
14n/a# Python imports
15n/aimport os
16n/aimport sys
17n/aimport logging
18n/aimport operator
19n/aimport collections
20n/aimport io
21n/afrom itertools import chain
22n/a
23n/a# Local imports
24n/afrom .pgen2 import driver, tokenize, token
25n/afrom .fixer_util import find_root
26n/afrom . import pytree, pygram
27n/afrom . import btm_matcher as bm
28n/a
29n/a
30n/adef get_all_fix_names(fixer_pkg, remove_prefix=True):
31n/a """Return a sorted list of all available fix names in the given package."""
32n/a pkg = __import__(fixer_pkg, [], [], ["*"])
33n/a fixer_dir = os.path.dirname(pkg.__file__)
34n/a fix_names = []
35n/a for name in sorted(os.listdir(fixer_dir)):
36n/a if name.startswith("fix_") and name.endswith(".py"):
37n/a if remove_prefix:
38n/a name = name[4:]
39n/a fix_names.append(name[:-3])
40n/a return fix_names
41n/a
42n/a
43n/aclass _EveryNode(Exception):
44n/a pass
45n/a
46n/a
47n/adef _get_head_types(pat):
48n/a """ Accepts a pytree Pattern Node and returns a set
49n/a of the pattern types which will match first. """
50n/a
51n/a if isinstance(pat, (pytree.NodePattern, pytree.LeafPattern)):
52n/a # NodePatters must either have no type and no content
53n/a # or a type and content -- so they don't get any farther
54n/a # Always return leafs
55n/a if pat.type is None:
56n/a raise _EveryNode
57n/a return {pat.type}
58n/a
59n/a if isinstance(pat, pytree.NegatedPattern):
60n/a if pat.content:
61n/a return _get_head_types(pat.content)
62n/a raise _EveryNode # Negated Patterns don't have a type
63n/a
64n/a if isinstance(pat, pytree.WildcardPattern):
65n/a # Recurse on each node in content
66n/a r = set()
67n/a for p in pat.content:
68n/a for x in p:
69n/a r.update(_get_head_types(x))
70n/a return r
71n/a
72n/a raise Exception("Oh no! I don't understand pattern %s" %(pat))
73n/a
74n/a
75n/adef _get_headnode_dict(fixer_list):
76n/a """ Accepts a list of fixers and returns a dictionary
77n/a of head node type --> fixer list. """
78n/a head_nodes = collections.defaultdict(list)
79n/a every = []
80n/a for fixer in fixer_list:
81n/a if fixer.pattern:
82n/a try:
83n/a heads = _get_head_types(fixer.pattern)
84n/a except _EveryNode:
85n/a every.append(fixer)
86n/a else:
87n/a for node_type in heads:
88n/a head_nodes[node_type].append(fixer)
89n/a else:
90n/a if fixer._accept_type is not None:
91n/a head_nodes[fixer._accept_type].append(fixer)
92n/a else:
93n/a every.append(fixer)
94n/a for node_type in chain(pygram.python_grammar.symbol2number.values(),
95n/a pygram.python_grammar.tokens):
96n/a head_nodes[node_type].extend(every)
97n/a return dict(head_nodes)
98n/a
99n/a
100n/adef get_fixers_from_package(pkg_name):
101n/a """
102n/a Return the fully qualified names for fixers in the package pkg_name.
103n/a """
104n/a return [pkg_name + "." + fix_name
105n/a for fix_name in get_all_fix_names(pkg_name, False)]
106n/a
107n/adef _identity(obj):
108n/a return obj
109n/a
110n/aif sys.version_info < (3, 0):
111n/a import codecs
112n/a _open_with_encoding = codecs.open
113n/a # codecs.open doesn't translate newlines sadly.
114n/a def _from_system_newlines(input):
115n/a return input.replace("\r\n", "\n")
116n/a def _to_system_newlines(input):
117n/a if os.linesep != "\n":
118n/a return input.replace("\n", os.linesep)
119n/a else:
120n/a return input
121n/aelse:
122n/a _open_with_encoding = open
123n/a _from_system_newlines = _identity
124n/a _to_system_newlines = _identity
125n/a
126n/a
127n/adef _detect_future_features(source):
128n/a have_docstring = False
129n/a gen = tokenize.generate_tokens(io.StringIO(source).readline)
130n/a def advance():
131n/a tok = next(gen)
132n/a return tok[0], tok[1]
133n/a ignore = frozenset({token.NEWLINE, tokenize.NL, token.COMMENT})
134n/a features = set()
135n/a try:
136n/a while True:
137n/a tp, value = advance()
138n/a if tp in ignore:
139n/a continue
140n/a elif tp == token.STRING:
141n/a if have_docstring:
142n/a break
143n/a have_docstring = True
144n/a elif tp == token.NAME and value == "from":
145n/a tp, value = advance()
146n/a if tp != token.NAME or value != "__future__":
147n/a break
148n/a tp, value = advance()
149n/a if tp != token.NAME or value != "import":
150n/a break
151n/a tp, value = advance()
152n/a if tp == token.OP and value == "(":
153n/a tp, value = advance()
154n/a while tp == token.NAME:
155n/a features.add(value)
156n/a tp, value = advance()
157n/a if tp != token.OP or value != ",":
158n/a break
159n/a tp, value = advance()
160n/a else:
161n/a break
162n/a except StopIteration:
163n/a pass
164n/a return frozenset(features)
165n/a
166n/a
167n/aclass FixerError(Exception):
168n/a """A fixer could not be loaded."""
169n/a
170n/a
171n/aclass RefactoringTool(object):
172n/a
173n/a _default_options = {"print_function" : False,
174n/a "write_unchanged_files" : False}
175n/a
176n/a CLASS_PREFIX = "Fix" # The prefix for fixer classes
177n/a FILE_PREFIX = "fix_" # The prefix for modules with a fixer within
178n/a
179n/a def __init__(self, fixer_names, options=None, explicit=None):
180n/a """Initializer.
181n/a
182n/a Args:
183n/a fixer_names: a list of fixers to import
184n/a options: a dict with configuration.
185n/a explicit: a list of fixers to run even if they are explicit.
186n/a """
187n/a self.fixers = fixer_names
188n/a self.explicit = explicit or []
189n/a self.options = self._default_options.copy()
190n/a if options is not None:
191n/a self.options.update(options)
192n/a if self.options["print_function"]:
193n/a self.grammar = pygram.python_grammar_no_print_statement
194n/a else:
195n/a self.grammar = pygram.python_grammar
196n/a # When this is True, the refactor*() methods will call write_file() for
197n/a # files processed even if they were not changed during refactoring. If
198n/a # and only if the refactor method's write parameter was True.
199n/a self.write_unchanged_files = self.options.get("write_unchanged_files")
200n/a self.errors = []
201n/a self.logger = logging.getLogger("RefactoringTool")
202n/a self.fixer_log = []
203n/a self.wrote = False
204n/a self.driver = driver.Driver(self.grammar,
205n/a convert=pytree.convert,
206n/a logger=self.logger)
207n/a self.pre_order, self.post_order = self.get_fixers()
208n/a
209n/a
210n/a self.files = [] # List of files that were or should be modified
211n/a
212n/a self.BM = bm.BottomMatcher()
213n/a self.bmi_pre_order = [] # Bottom Matcher incompatible fixers
214n/a self.bmi_post_order = []
215n/a
216n/a for fixer in chain(self.post_order, self.pre_order):
217n/a if fixer.BM_compatible:
218n/a self.BM.add_fixer(fixer)
219n/a # remove fixers that will be handled by the bottom-up
220n/a # matcher
221n/a elif fixer in self.pre_order:
222n/a self.bmi_pre_order.append(fixer)
223n/a elif fixer in self.post_order:
224n/a self.bmi_post_order.append(fixer)
225n/a
226n/a self.bmi_pre_order_heads = _get_headnode_dict(self.bmi_pre_order)
227n/a self.bmi_post_order_heads = _get_headnode_dict(self.bmi_post_order)
228n/a
229n/a
230n/a
231n/a def get_fixers(self):
232n/a """Inspects the options to load the requested patterns and handlers.
233n/a
234n/a Returns:
235n/a (pre_order, post_order), where pre_order is the list of fixers that
236n/a want a pre-order AST traversal, and post_order is the list that want
237n/a post-order traversal.
238n/a """
239n/a pre_order_fixers = []
240n/a post_order_fixers = []
241n/a for fix_mod_path in self.fixers:
242n/a mod = __import__(fix_mod_path, {}, {}, ["*"])
243n/a fix_name = fix_mod_path.rsplit(".", 1)[-1]
244n/a if fix_name.startswith(self.FILE_PREFIX):
245n/a fix_name = fix_name[len(self.FILE_PREFIX):]
246n/a parts = fix_name.split("_")
247n/a class_name = self.CLASS_PREFIX + "".join([p.title() for p in parts])
248n/a try:
249n/a fix_class = getattr(mod, class_name)
250n/a except AttributeError:
251n/a raise FixerError("Can't find %s.%s" % (fix_name, class_name))
252n/a fixer = fix_class(self.options, self.fixer_log)
253n/a if fixer.explicit and self.explicit is not True and \
254n/a fix_mod_path not in self.explicit:
255n/a self.log_message("Skipping optional fixer: %s", fix_name)
256n/a continue
257n/a
258n/a self.log_debug("Adding transformation: %s", fix_name)
259n/a if fixer.order == "pre":
260n/a pre_order_fixers.append(fixer)
261n/a elif fixer.order == "post":
262n/a post_order_fixers.append(fixer)
263n/a else:
264n/a raise FixerError("Illegal fixer order: %r" % fixer.order)
265n/a
266n/a key_func = operator.attrgetter("run_order")
267n/a pre_order_fixers.sort(key=key_func)
268n/a post_order_fixers.sort(key=key_func)
269n/a return (pre_order_fixers, post_order_fixers)
270n/a
271n/a def log_error(self, msg, *args, **kwds):
272n/a """Called when an error occurs."""
273n/a raise
274n/a
275n/a def log_message(self, msg, *args):
276n/a """Hook to log a message."""
277n/a if args:
278n/a msg = msg % args
279n/a self.logger.info(msg)
280n/a
281n/a def log_debug(self, msg, *args):
282n/a if args:
283n/a msg = msg % args
284n/a self.logger.debug(msg)
285n/a
286n/a def print_output(self, old_text, new_text, filename, equal):
287n/a """Called with the old version, new version, and filename of a
288n/a refactored file."""
289n/a pass
290n/a
291n/a def refactor(self, items, write=False, doctests_only=False):
292n/a """Refactor a list of files and directories."""
293n/a
294n/a for dir_or_file in items:
295n/a if os.path.isdir(dir_or_file):
296n/a self.refactor_dir(dir_or_file, write, doctests_only)
297n/a else:
298n/a self.refactor_file(dir_or_file, write, doctests_only)
299n/a
300n/a def refactor_dir(self, dir_name, write=False, doctests_only=False):
301n/a """Descends down a directory and refactor every Python file found.
302n/a
303n/a Python files are assumed to have a .py extension.
304n/a
305n/a Files and subdirectories starting with '.' are skipped.
306n/a """
307n/a py_ext = os.extsep + "py"
308n/a for dirpath, dirnames, filenames in os.walk(dir_name):
309n/a self.log_debug("Descending into %s", dirpath)
310n/a dirnames.sort()
311n/a filenames.sort()
312n/a for name in filenames:
313n/a if (not name.startswith(".") and
314n/a os.path.splitext(name)[1] == py_ext):
315n/a fullname = os.path.join(dirpath, name)
316n/a self.refactor_file(fullname, write, doctests_only)
317n/a # Modify dirnames in-place to remove subdirs with leading dots
318n/a dirnames[:] = [dn for dn in dirnames if not dn.startswith(".")]
319n/a
320n/a def _read_python_source(self, filename):
321n/a """
322n/a Do our best to decode a Python source file correctly.
323n/a """
324n/a try:
325n/a f = open(filename, "rb")
326n/a except OSError as err:
327n/a self.log_error("Can't open %s: %s", filename, err)
328n/a return None, None
329n/a try:
330n/a encoding = tokenize.detect_encoding(f.readline)[0]
331n/a finally:
332n/a f.close()
333n/a with _open_with_encoding(filename, "r", encoding=encoding) as f:
334n/a return _from_system_newlines(f.read()), encoding
335n/a
336n/a def refactor_file(self, filename, write=False, doctests_only=False):
337n/a """Refactors a file."""
338n/a input, encoding = self._read_python_source(filename)
339n/a if input is None:
340n/a # Reading the file failed.
341n/a return
342n/a input += "\n" # Silence certain parse errors
343n/a if doctests_only:
344n/a self.log_debug("Refactoring doctests in %s", filename)
345n/a output = self.refactor_docstring(input, filename)
346n/a if self.write_unchanged_files or output != input:
347n/a self.processed_file(output, filename, input, write, encoding)
348n/a else:
349n/a self.log_debug("No doctest changes in %s", filename)
350n/a else:
351n/a tree = self.refactor_string(input, filename)
352n/a if self.write_unchanged_files or (tree and tree.was_changed):
353n/a # The [:-1] is to take off the \n we added earlier
354n/a self.processed_file(str(tree)[:-1], filename,
355n/a write=write, encoding=encoding)
356n/a else:
357n/a self.log_debug("No changes in %s", filename)
358n/a
359n/a def refactor_string(self, data, name):
360n/a """Refactor a given input string.
361n/a
362n/a Args:
363n/a data: a string holding the code to be refactored.
364n/a name: a human-readable name for use in error/log messages.
365n/a
366n/a Returns:
367n/a An AST corresponding to the refactored input stream; None if
368n/a there were errors during the parse.
369n/a """
370n/a features = _detect_future_features(data)
371n/a if "print_function" in features:
372n/a self.driver.grammar = pygram.python_grammar_no_print_statement
373n/a try:
374n/a tree = self.driver.parse_string(data)
375n/a except Exception as err:
376n/a self.log_error("Can't parse %s: %s: %s",
377n/a name, err.__class__.__name__, err)
378n/a return
379n/a finally:
380n/a self.driver.grammar = self.grammar
381n/a tree.future_features = features
382n/a self.log_debug("Refactoring %s", name)
383n/a self.refactor_tree(tree, name)
384n/a return tree
385n/a
386n/a def refactor_stdin(self, doctests_only=False):
387n/a input = sys.stdin.read()
388n/a if doctests_only:
389n/a self.log_debug("Refactoring doctests in stdin")
390n/a output = self.refactor_docstring(input, "<stdin>")
391n/a if self.write_unchanged_files or output != input:
392n/a self.processed_file(output, "<stdin>", input)
393n/a else:
394n/a self.log_debug("No doctest changes in stdin")
395n/a else:
396n/a tree = self.refactor_string(input, "<stdin>")
397n/a if self.write_unchanged_files or (tree and tree.was_changed):
398n/a self.processed_file(str(tree), "<stdin>", input)
399n/a else:
400n/a self.log_debug("No changes in stdin")
401n/a
402n/a def refactor_tree(self, tree, name):
403n/a """Refactors a parse tree (modifying the tree in place).
404n/a
405n/a For compatible patterns the bottom matcher module is
406n/a used. Otherwise the tree is traversed node-to-node for
407n/a matches.
408n/a
409n/a Args:
410n/a tree: a pytree.Node instance representing the root of the tree
411n/a to be refactored.
412n/a name: a human-readable name for this tree.
413n/a
414n/a Returns:
415n/a True if the tree was modified, False otherwise.
416n/a """
417n/a
418n/a for fixer in chain(self.pre_order, self.post_order):
419n/a fixer.start_tree(tree, name)
420n/a
421n/a #use traditional matching for the incompatible fixers
422n/a self.traverse_by(self.bmi_pre_order_heads, tree.pre_order())
423n/a self.traverse_by(self.bmi_post_order_heads, tree.post_order())
424n/a
425n/a # obtain a set of candidate nodes
426n/a match_set = self.BM.run(tree.leaves())
427n/a
428n/a while any(match_set.values()):
429n/a for fixer in self.BM.fixers:
430n/a if fixer in match_set and match_set[fixer]:
431n/a #sort by depth; apply fixers from bottom(of the AST) to top
432n/a match_set[fixer].sort(key=pytree.Base.depth, reverse=True)
433n/a
434n/a if fixer.keep_line_order:
435n/a #some fixers(eg fix_imports) must be applied
436n/a #with the original file's line order
437n/a match_set[fixer].sort(key=pytree.Base.get_lineno)
438n/a
439n/a for node in list(match_set[fixer]):
440n/a if node in match_set[fixer]:
441n/a match_set[fixer].remove(node)
442n/a
443n/a try:
444n/a find_root(node)
445n/a except ValueError:
446n/a # this node has been cut off from a
447n/a # previous transformation ; skip
448n/a continue
449n/a
450n/a if node.fixers_applied and fixer in node.fixers_applied:
451n/a # do not apply the same fixer again
452n/a continue
453n/a
454n/a results = fixer.match(node)
455n/a
456n/a if results:
457n/a new = fixer.transform(node, results)
458n/a if new is not None:
459n/a node.replace(new)
460n/a #new.fixers_applied.append(fixer)
461n/a for node in new.post_order():
462n/a # do not apply the fixer again to
463n/a # this or any subnode
464n/a if not node.fixers_applied:
465n/a node.fixers_applied = []
466n/a node.fixers_applied.append(fixer)
467n/a
468n/a # update the original match set for
469n/a # the added code
470n/a new_matches = self.BM.run(new.leaves())
471n/a for fxr in new_matches:
472n/a if not fxr in match_set:
473n/a match_set[fxr]=[]
474n/a
475n/a match_set[fxr].extend(new_matches[fxr])
476n/a
477n/a for fixer in chain(self.pre_order, self.post_order):
478n/a fixer.finish_tree(tree, name)
479n/a return tree.was_changed
480n/a
481n/a def traverse_by(self, fixers, traversal):
482n/a """Traverse an AST, applying a set of fixers to each node.
483n/a
484n/a This is a helper method for refactor_tree().
485n/a
486n/a Args:
487n/a fixers: a list of fixer instances.
488n/a traversal: a generator that yields AST nodes.
489n/a
490n/a Returns:
491n/a None
492n/a """
493n/a if not fixers:
494n/a return
495n/a for node in traversal:
496n/a for fixer in fixers[node.type]:
497n/a results = fixer.match(node)
498n/a if results:
499n/a new = fixer.transform(node, results)
500n/a if new is not None:
501n/a node.replace(new)
502n/a node = new
503n/a
504n/a def processed_file(self, new_text, filename, old_text=None, write=False,
505n/a encoding=None):
506n/a """
507n/a Called when a file has been refactored and there may be changes.
508n/a """
509n/a self.files.append(filename)
510n/a if old_text is None:
511n/a old_text = self._read_python_source(filename)[0]
512n/a if old_text is None:
513n/a return
514n/a equal = old_text == new_text
515n/a self.print_output(old_text, new_text, filename, equal)
516n/a if equal:
517n/a self.log_debug("No changes to %s", filename)
518n/a if not self.write_unchanged_files:
519n/a return
520n/a if write:
521n/a self.write_file(new_text, filename, old_text, encoding)
522n/a else:
523n/a self.log_debug("Not writing changes to %s", filename)
524n/a
525n/a def write_file(self, new_text, filename, old_text, encoding=None):
526n/a """Writes a string to a file.
527n/a
528n/a It first shows a unified diff between the old text and the new text, and
529n/a then rewrites the file; the latter is only done if the write option is
530n/a set.
531n/a """
532n/a try:
533n/a f = _open_with_encoding(filename, "w", encoding=encoding)
534n/a except OSError as err:
535n/a self.log_error("Can't create %s: %s", filename, err)
536n/a return
537n/a try:
538n/a f.write(_to_system_newlines(new_text))
539n/a except OSError as err:
540n/a self.log_error("Can't write %s: %s", filename, err)
541n/a finally:
542n/a f.close()
543n/a self.log_debug("Wrote changes to %s", filename)
544n/a self.wrote = True
545n/a
546n/a PS1 = ">>> "
547n/a PS2 = "... "
548n/a
549n/a def refactor_docstring(self, input, filename):
550n/a """Refactors a docstring, looking for doctests.
551n/a
552n/a This returns a modified version of the input string. It looks
553n/a for doctests, which start with a ">>>" prompt, and may be
554n/a continued with "..." prompts, as long as the "..." is indented
555n/a the same as the ">>>".
556n/a
557n/a (Unfortunately we can't use the doctest module's parser,
558n/a since, like most parsers, it is not geared towards preserving
559n/a the original source.)
560n/a """
561n/a result = []
562n/a block = None
563n/a block_lineno = None
564n/a indent = None
565n/a lineno = 0
566n/a for line in input.splitlines(keepends=True):
567n/a lineno += 1
568n/a if line.lstrip().startswith(self.PS1):
569n/a if block is not None:
570n/a result.extend(self.refactor_doctest(block, block_lineno,
571n/a indent, filename))
572n/a block_lineno = lineno
573n/a block = [line]
574n/a i = line.find(self.PS1)
575n/a indent = line[:i]
576n/a elif (indent is not None and
577n/a (line.startswith(indent + self.PS2) or
578n/a line == indent + self.PS2.rstrip() + "\n")):
579n/a block.append(line)
580n/a else:
581n/a if block is not None:
582n/a result.extend(self.refactor_doctest(block, block_lineno,
583n/a indent, filename))
584n/a block = None
585n/a indent = None
586n/a result.append(line)
587n/a if block is not None:
588n/a result.extend(self.refactor_doctest(block, block_lineno,
589n/a indent, filename))
590n/a return "".join(result)
591n/a
592n/a def refactor_doctest(self, block, lineno, indent, filename):
593n/a """Refactors one doctest.
594n/a
595n/a A doctest is given as a block of lines, the first of which starts
596n/a with ">>>" (possibly indented), while the remaining lines start
597n/a with "..." (identically indented).
598n/a
599n/a """
600n/a try:
601n/a tree = self.parse_block(block, lineno, indent)
602n/a except Exception as err:
603n/a if self.logger.isEnabledFor(logging.DEBUG):
604n/a for line in block:
605n/a self.log_debug("Source: %s", line.rstrip("\n"))
606n/a self.log_error("Can't parse docstring in %s line %s: %s: %s",
607n/a filename, lineno, err.__class__.__name__, err)
608n/a return block
609n/a if self.refactor_tree(tree, filename):
610n/a new = str(tree).splitlines(keepends=True)
611n/a # Undo the adjustment of the line numbers in wrap_toks() below.
612n/a clipped, new = new[:lineno-1], new[lineno-1:]
613n/a assert clipped == ["\n"] * (lineno-1), clipped
614n/a if not new[-1].endswith("\n"):
615n/a new[-1] += "\n"
616n/a block = [indent + self.PS1 + new.pop(0)]
617n/a if new:
618n/a block += [indent + self.PS2 + line for line in new]
619n/a return block
620n/a
621n/a def summarize(self):
622n/a if self.wrote:
623n/a were = "were"
624n/a else:
625n/a were = "need to be"
626n/a if not self.files:
627n/a self.log_message("No files %s modified.", were)
628n/a else:
629n/a self.log_message("Files that %s modified:", were)
630n/a for file in self.files:
631n/a self.log_message(file)
632n/a if self.fixer_log:
633n/a self.log_message("Warnings/messages while refactoring:")
634n/a for message in self.fixer_log:
635n/a self.log_message(message)
636n/a if self.errors:
637n/a if len(self.errors) == 1:
638n/a self.log_message("There was 1 error:")
639n/a else:
640n/a self.log_message("There were %d errors:", len(self.errors))
641n/a for msg, args, kwds in self.errors:
642n/a self.log_message(msg, *args, **kwds)
643n/a
644n/a def parse_block(self, block, lineno, indent):
645n/a """Parses a block into a tree.
646n/a
647n/a This is necessary to get correct line number / offset information
648n/a in the parser diagnostics and embedded into the parse tree.
649n/a """
650n/a tree = self.driver.parse_tokens(self.wrap_toks(block, lineno, indent))
651n/a tree.future_features = frozenset()
652n/a return tree
653n/a
654n/a def wrap_toks(self, block, lineno, indent):
655n/a """Wraps a tokenize stream to systematically modify start/end."""
656n/a tokens = tokenize.generate_tokens(self.gen_lines(block, indent).__next__)
657n/a for type, value, (line0, col0), (line1, col1), line_text in tokens:
658n/a line0 += lineno - 1
659n/a line1 += lineno - 1
660n/a # Don't bother updating the columns; this is too complicated
661n/a # since line_text would also have to be updated and it would
662n/a # still break for tokens spanning lines. Let the user guess
663n/a # that the column numbers for doctests are relative to the
664n/a # end of the prompt string (PS1 or PS2).
665n/a yield type, value, (line0, col0), (line1, col1), line_text
666n/a
667n/a
668n/a def gen_lines(self, block, indent):
669n/a """Generates lines as expected by tokenize from a list of lines.
670n/a
671n/a This strips the first len(indent + self.PS1) characters off each line.
672n/a """
673n/a prefix1 = indent + self.PS1
674n/a prefix2 = indent + self.PS2
675n/a prefix = prefix1
676n/a for line in block:
677n/a if line.startswith(prefix):
678n/a yield line[len(prefix):]
679n/a elif line == prefix.rstrip() + "\n":
680n/a yield "\n"
681n/a else:
682n/a raise AssertionError("line=%r, prefix=%r" % (line, prefix))
683n/a prefix = prefix2
684n/a while True:
685n/a yield ""
686n/a
687n/a
688n/aclass MultiprocessingUnsupported(Exception):
689n/a pass
690n/a
691n/a
692n/aclass MultiprocessRefactoringTool(RefactoringTool):
693n/a
694n/a def __init__(self, *args, **kwargs):
695n/a super(MultiprocessRefactoringTool, self).__init__(*args, **kwargs)
696n/a self.queue = None
697n/a self.output_lock = None
698n/a
699n/a def refactor(self, items, write=False, doctests_only=False,
700n/a num_processes=1):
701n/a if num_processes == 1:
702n/a return super(MultiprocessRefactoringTool, self).refactor(
703n/a items, write, doctests_only)
704n/a try:
705n/a import multiprocessing
706n/a except ImportError:
707n/a raise MultiprocessingUnsupported
708n/a if self.queue is not None:
709n/a raise RuntimeError("already doing multiple processes")
710n/a self.queue = multiprocessing.JoinableQueue()
711n/a self.output_lock = multiprocessing.Lock()
712n/a processes = [multiprocessing.Process(target=self._child)
713n/a for i in range(num_processes)]
714n/a try:
715n/a for p in processes:
716n/a p.start()
717n/a super(MultiprocessRefactoringTool, self).refactor(items, write,
718n/a doctests_only)
719n/a finally:
720n/a self.queue.join()
721n/a for i in range(num_processes):
722n/a self.queue.put(None)
723n/a for p in processes:
724n/a if p.is_alive():
725n/a p.join()
726n/a self.queue = None
727n/a
728n/a def _child(self):
729n/a task = self.queue.get()
730n/a while task is not None:
731n/a args, kwargs = task
732n/a try:
733n/a super(MultiprocessRefactoringTool, self).refactor_file(
734n/a *args, **kwargs)
735n/a finally:
736n/a self.queue.task_done()
737n/a task = self.queue.get()
738n/a
739n/a def refactor_file(self, *args, **kwargs):
740n/a if self.queue is not None:
741n/a self.queue.put((args, kwargs))
742n/a else:
743n/a return super(MultiprocessRefactoringTool, self).refactor_file(
744n/a *args, **kwargs)