ยปCore Development>Code coverage>Lib/distutils/util.py

Python code coverage for Lib/distutils/util.py

#countcontent
1n/a"""distutils.util
2n/a
3n/aMiscellaneous utility functions -- anything that doesn't fit into
4n/aone of the other *util.py modules.
5n/a"""
6n/a
7n/aimport os
8n/aimport re
9n/aimport importlib.util
10n/aimport string
11n/aimport sys
12n/afrom distutils.errors import DistutilsPlatformError
13n/afrom distutils.dep_util import newer
14n/afrom distutils.spawn import spawn
15n/afrom distutils import log
16n/afrom distutils.errors import DistutilsByteCompileError
17n/a
18n/adef get_platform ():
19n/a """Return a string that identifies the current platform. This is used
20n/a mainly to distinguish platform-specific build directories and
21n/a platform-specific built distributions. Typically includes the OS name
22n/a and version and the architecture (as supplied by 'os.uname()'),
23n/a although the exact information included depends on the OS; eg. for IRIX
24n/a the architecture isn't particularly important (IRIX only runs on SGI
25n/a hardware), but for Linux the kernel version isn't particularly
26n/a important.
27n/a
28n/a Examples of returned values:
29n/a linux-i586
30n/a linux-alpha (?)
31n/a solaris-2.6-sun4u
32n/a irix-5.3
33n/a irix64-6.2
34n/a
35n/a Windows will return one of:
36n/a win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc)
37n/a win-ia64 (64bit Windows on Itanium)
38n/a win32 (all others - specifically, sys.platform is returned)
39n/a
40n/a For other non-POSIX platforms, currently just returns 'sys.platform'.
41n/a """
42n/a if os.name == 'nt':
43n/a # sniff sys.version for architecture.
44n/a prefix = " bit ("
45n/a i = sys.version.find(prefix)
46n/a if i == -1:
47n/a return sys.platform
48n/a j = sys.version.find(")", i)
49n/a look = sys.version[i+len(prefix):j].lower()
50n/a if look == 'amd64':
51n/a return 'win-amd64'
52n/a if look == 'itanium':
53n/a return 'win-ia64'
54n/a return sys.platform
55n/a
56n/a # Set for cross builds explicitly
57n/a if "_PYTHON_HOST_PLATFORM" in os.environ:
58n/a return os.environ["_PYTHON_HOST_PLATFORM"]
59n/a
60n/a if os.name != "posix" or not hasattr(os, 'uname'):
61n/a # XXX what about the architecture? NT is Intel or Alpha,
62n/a # Mac OS is M68k or PPC, etc.
63n/a return sys.platform
64n/a
65n/a # Try to distinguish various flavours of Unix
66n/a
67n/a (osname, host, release, version, machine) = os.uname()
68n/a
69n/a # Convert the OS name to lowercase, remove '/' characters
70n/a # (to accommodate BSD/OS), and translate spaces (for "Power Macintosh")
71n/a osname = osname.lower().replace('/', '')
72n/a machine = machine.replace(' ', '_')
73n/a machine = machine.replace('/', '-')
74n/a
75n/a if osname[:5] == "linux":
76n/a # At least on Linux/Intel, 'machine' is the processor --
77n/a # i386, etc.
78n/a # XXX what about Alpha, SPARC, etc?
79n/a return "%s-%s" % (osname, machine)
80n/a elif osname[:5] == "sunos":
81n/a if release[0] >= "5": # SunOS 5 == Solaris 2
82n/a osname = "solaris"
83n/a release = "%d.%s" % (int(release[0]) - 3, release[2:])
84n/a # We can't use "platform.architecture()[0]" because a
85n/a # bootstrap problem. We use a dict to get an error
86n/a # if some suspicious happens.
87n/a bitness = {2147483647:"32bit", 9223372036854775807:"64bit"}
88n/a machine += ".%s" % bitness[sys.maxsize]
89n/a # fall through to standard osname-release-machine representation
90n/a elif osname[:4] == "irix": # could be "irix64"!
91n/a return "%s-%s" % (osname, release)
92n/a elif osname[:3] == "aix":
93n/a return "%s-%s.%s" % (osname, version, release)
94n/a elif osname[:6] == "cygwin":
95n/a osname = "cygwin"
96n/a rel_re = re.compile (r'[\d.]+', re.ASCII)
97n/a m = rel_re.match(release)
98n/a if m:
99n/a release = m.group()
100n/a elif osname[:6] == "darwin":
101n/a import _osx_support, distutils.sysconfig
102n/a osname, release, machine = _osx_support.get_platform_osx(
103n/a distutils.sysconfig.get_config_vars(),
104n/a osname, release, machine)
105n/a
106n/a return "%s-%s-%s" % (osname, release, machine)
107n/a
108n/a# get_platform ()
109n/a
110n/a
111n/adef convert_path (pathname):
112n/a """Return 'pathname' as a name that will work on the native filesystem,
113n/a i.e. split it on '/' and put it back together again using the current
114n/a directory separator. Needed because filenames in the setup script are
115n/a always supplied in Unix style, and have to be converted to the local
116n/a convention before we can actually use them in the filesystem. Raises
117n/a ValueError on non-Unix-ish systems if 'pathname' either starts or
118n/a ends with a slash.
119n/a """
120n/a if os.sep == '/':
121n/a return pathname
122n/a if not pathname:
123n/a return pathname
124n/a if pathname[0] == '/':
125n/a raise ValueError("path '%s' cannot be absolute" % pathname)
126n/a if pathname[-1] == '/':
127n/a raise ValueError("path '%s' cannot end with '/'" % pathname)
128n/a
129n/a paths = pathname.split('/')
130n/a while '.' in paths:
131n/a paths.remove('.')
132n/a if not paths:
133n/a return os.curdir
134n/a return os.path.join(*paths)
135n/a
136n/a# convert_path ()
137n/a
138n/a
139n/adef change_root (new_root, pathname):
140n/a """Return 'pathname' with 'new_root' prepended. If 'pathname' is
141n/a relative, this is equivalent to "os.path.join(new_root,pathname)".
142n/a Otherwise, it requires making 'pathname' relative and then joining the
143n/a two, which is tricky on DOS/Windows and Mac OS.
144n/a """
145n/a if os.name == 'posix':
146n/a if not os.path.isabs(pathname):
147n/a return os.path.join(new_root, pathname)
148n/a else:
149n/a return os.path.join(new_root, pathname[1:])
150n/a
151n/a elif os.name == 'nt':
152n/a (drive, path) = os.path.splitdrive(pathname)
153n/a if path[0] == '\\':
154n/a path = path[1:]
155n/a return os.path.join(new_root, path)
156n/a
157n/a else:
158n/a raise DistutilsPlatformError("nothing known about platform '%s'" % os.name)
159n/a
160n/a
161n/a_environ_checked = 0
162n/adef check_environ ():
163n/a """Ensure that 'os.environ' has all the environment variables we
164n/a guarantee that users can use in config files, command-line options,
165n/a etc. Currently this includes:
166n/a HOME - user's home directory (Unix only)
167n/a PLAT - description of the current platform, including hardware
168n/a and OS (see 'get_platform()')
169n/a """
170n/a global _environ_checked
171n/a if _environ_checked:
172n/a return
173n/a
174n/a if os.name == 'posix' and 'HOME' not in os.environ:
175n/a import pwd
176n/a os.environ['HOME'] = pwd.getpwuid(os.getuid())[5]
177n/a
178n/a if 'PLAT' not in os.environ:
179n/a os.environ['PLAT'] = get_platform()
180n/a
181n/a _environ_checked = 1
182n/a
183n/a
184n/adef subst_vars (s, local_vars):
185n/a """Perform shell/Perl-style variable substitution on 'string'. Every
186n/a occurrence of '$' followed by a name is considered a variable, and
187n/a variable is substituted by the value found in the 'local_vars'
188n/a dictionary, or in 'os.environ' if it's not in 'local_vars'.
189n/a 'os.environ' is first checked/augmented to guarantee that it contains
190n/a certain values: see 'check_environ()'. Raise ValueError for any
191n/a variables not found in either 'local_vars' or 'os.environ'.
192n/a """
193n/a check_environ()
194n/a def _subst (match, local_vars=local_vars):
195n/a var_name = match.group(1)
196n/a if var_name in local_vars:
197n/a return str(local_vars[var_name])
198n/a else:
199n/a return os.environ[var_name]
200n/a
201n/a try:
202n/a return re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s)
203n/a except KeyError as var:
204n/a raise ValueError("invalid variable '$%s'" % var)
205n/a
206n/a# subst_vars ()
207n/a
208n/a
209n/adef grok_environment_error (exc, prefix="error: "):
210n/a # Function kept for backward compatibility.
211n/a # Used to try clever things with EnvironmentErrors,
212n/a # but nowadays str(exception) produces good messages.
213n/a return prefix + str(exc)
214n/a
215n/a
216n/a# Needed by 'split_quoted()'
217n/a_wordchars_re = _squote_re = _dquote_re = None
218n/adef _init_regex():
219n/a global _wordchars_re, _squote_re, _dquote_re
220n/a _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace)
221n/a _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'")
222n/a _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"')
223n/a
224n/adef split_quoted (s):
225n/a """Split a string up according to Unix shell-like rules for quotes and
226n/a backslashes. In short: words are delimited by spaces, as long as those
227n/a spaces are not escaped by a backslash, or inside a quoted string.
228n/a Single and double quotes are equivalent, and the quote characters can
229n/a be backslash-escaped. The backslash is stripped from any two-character
230n/a escape sequence, leaving only the escaped character. The quote
231n/a characters are stripped from any quoted string. Returns a list of
232n/a words.
233n/a """
234n/a
235n/a # This is a nice algorithm for splitting up a single string, since it
236n/a # doesn't require character-by-character examination. It was a little
237n/a # bit of a brain-bender to get it working right, though...
238n/a if _wordchars_re is None: _init_regex()
239n/a
240n/a s = s.strip()
241n/a words = []
242n/a pos = 0
243n/a
244n/a while s:
245n/a m = _wordchars_re.match(s, pos)
246n/a end = m.end()
247n/a if end == len(s):
248n/a words.append(s[:end])
249n/a break
250n/a
251n/a if s[end] in string.whitespace: # unescaped, unquoted whitespace: now
252n/a words.append(s[:end]) # we definitely have a word delimiter
253n/a s = s[end:].lstrip()
254n/a pos = 0
255n/a
256n/a elif s[end] == '\\': # preserve whatever is being escaped;
257n/a # will become part of the current word
258n/a s = s[:end] + s[end+1:]
259n/a pos = end+1
260n/a
261n/a else:
262n/a if s[end] == "'": # slurp singly-quoted string
263n/a m = _squote_re.match(s, end)
264n/a elif s[end] == '"': # slurp doubly-quoted string
265n/a m = _dquote_re.match(s, end)
266n/a else:
267n/a raise RuntimeError("this can't happen (bad char '%c')" % s[end])
268n/a
269n/a if m is None:
270n/a raise ValueError("bad string (mismatched %s quotes?)" % s[end])
271n/a
272n/a (beg, end) = m.span()
273n/a s = s[:beg] + s[beg+1:end-1] + s[end:]
274n/a pos = m.end() - 2
275n/a
276n/a if pos >= len(s):
277n/a words.append(s)
278n/a break
279n/a
280n/a return words
281n/a
282n/a# split_quoted ()
283n/a
284n/a
285n/adef execute (func, args, msg=None, verbose=0, dry_run=0):
286n/a """Perform some action that affects the outside world (eg. by
287n/a writing to the filesystem). Such actions are special because they
288n/a are disabled by the 'dry_run' flag. This method takes care of all
289n/a that bureaucracy for you; all you have to do is supply the
290n/a function to call and an argument tuple for it (to embody the
291n/a "external action" being performed), and an optional message to
292n/a print.
293n/a """
294n/a if msg is None:
295n/a msg = "%s%r" % (func.__name__, args)
296n/a if msg[-2:] == ',)': # correct for singleton tuple
297n/a msg = msg[0:-2] + ')'
298n/a
299n/a log.info(msg)
300n/a if not dry_run:
301n/a func(*args)
302n/a
303n/a
304n/adef strtobool (val):
305n/a """Convert a string representation of truth to true (1) or false (0).
306n/a
307n/a True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
308n/a are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if
309n/a 'val' is anything else.
310n/a """
311n/a val = val.lower()
312n/a if val in ('y', 'yes', 't', 'true', 'on', '1'):
313n/a return 1
314n/a elif val in ('n', 'no', 'f', 'false', 'off', '0'):
315n/a return 0
316n/a else:
317n/a raise ValueError("invalid truth value %r" % (val,))
318n/a
319n/a
320n/adef byte_compile (py_files,
321n/a optimize=0, force=0,
322n/a prefix=None, base_dir=None,
323n/a verbose=1, dry_run=0,
324n/a direct=None):
325n/a """Byte-compile a collection of Python source files to .pyc
326n/a files in a __pycache__ subdirectory. 'py_files' is a list
327n/a of files to compile; any files that don't end in ".py" are silently
328n/a skipped. 'optimize' must be one of the following:
329n/a 0 - don't optimize
330n/a 1 - normal optimization (like "python -O")
331n/a 2 - extra optimization (like "python -OO")
332n/a If 'force' is true, all files are recompiled regardless of
333n/a timestamps.
334n/a
335n/a The source filename encoded in each bytecode file defaults to the
336n/a filenames listed in 'py_files'; you can modify these with 'prefix' and
337n/a 'basedir'. 'prefix' is a string that will be stripped off of each
338n/a source filename, and 'base_dir' is a directory name that will be
339n/a prepended (after 'prefix' is stripped). You can supply either or both
340n/a (or neither) of 'prefix' and 'base_dir', as you wish.
341n/a
342n/a If 'dry_run' is true, doesn't actually do anything that would
343n/a affect the filesystem.
344n/a
345n/a Byte-compilation is either done directly in this interpreter process
346n/a with the standard py_compile module, or indirectly by writing a
347n/a temporary script and executing it. Normally, you should let
348n/a 'byte_compile()' figure out to use direct compilation or not (see
349n/a the source for details). The 'direct' flag is used by the script
350n/a generated in indirect mode; unless you know what you're doing, leave
351n/a it set to None.
352n/a """
353n/a
354n/a # Late import to fix a bootstrap issue: _posixsubprocess is built by
355n/a # setup.py, but setup.py uses distutils.
356n/a import subprocess
357n/a
358n/a # nothing is done if sys.dont_write_bytecode is True
359n/a if sys.dont_write_bytecode:
360n/a raise DistutilsByteCompileError('byte-compiling is disabled.')
361n/a
362n/a # First, if the caller didn't force us into direct or indirect mode,
363n/a # figure out which mode we should be in. We take a conservative
364n/a # approach: choose direct mode *only* if the current interpreter is
365n/a # in debug mode and optimize is 0. If we're not in debug mode (-O
366n/a # or -OO), we don't know which level of optimization this
367n/a # interpreter is running with, so we can't do direct
368n/a # byte-compilation and be certain that it's the right thing. Thus,
369n/a # always compile indirectly if the current interpreter is in either
370n/a # optimize mode, or if either optimization level was requested by
371n/a # the caller.
372n/a if direct is None:
373n/a direct = (__debug__ and optimize == 0)
374n/a
375n/a # "Indirect" byte-compilation: write a temporary script and then
376n/a # run it with the appropriate flags.
377n/a if not direct:
378n/a try:
379n/a from tempfile import mkstemp
380n/a (script_fd, script_name) = mkstemp(".py")
381n/a except ImportError:
382n/a from tempfile import mktemp
383n/a (script_fd, script_name) = None, mktemp(".py")
384n/a log.info("writing byte-compilation script '%s'", script_name)
385n/a if not dry_run:
386n/a if script_fd is not None:
387n/a script = os.fdopen(script_fd, "w")
388n/a else:
389n/a script = open(script_name, "w")
390n/a
391n/a script.write("""\
392n/afrom distutils.util import byte_compile
393n/afiles = [
394n/a""")
395n/a
396n/a # XXX would be nice to write absolute filenames, just for
397n/a # safety's sake (script should be more robust in the face of
398n/a # chdir'ing before running it). But this requires abspath'ing
399n/a # 'prefix' as well, and that breaks the hack in build_lib's
400n/a # 'byte_compile()' method that carefully tacks on a trailing
401n/a # slash (os.sep really) to make sure the prefix here is "just
402n/a # right". This whole prefix business is rather delicate -- the
403n/a # problem is that it's really a directory, but I'm treating it
404n/a # as a dumb string, so trailing slashes and so forth matter.
405n/a
406n/a #py_files = map(os.path.abspath, py_files)
407n/a #if prefix:
408n/a # prefix = os.path.abspath(prefix)
409n/a
410n/a script.write(",\n".join(map(repr, py_files)) + "]\n")
411n/a script.write("""
412n/abyte_compile(files, optimize=%r, force=%r,
413n/a prefix=%r, base_dir=%r,
414n/a verbose=%r, dry_run=0,
415n/a direct=1)
416n/a""" % (optimize, force, prefix, base_dir, verbose))
417n/a
418n/a script.close()
419n/a
420n/a cmd = [sys.executable]
421n/a cmd.extend(subprocess._optim_args_from_interpreter_flags())
422n/a cmd.append(script_name)
423n/a spawn(cmd, dry_run=dry_run)
424n/a execute(os.remove, (script_name,), "removing %s" % script_name,
425n/a dry_run=dry_run)
426n/a
427n/a # "Direct" byte-compilation: use the py_compile module to compile
428n/a # right here, right now. Note that the script generated in indirect
429n/a # mode simply calls 'byte_compile()' in direct mode, a weird sort of
430n/a # cross-process recursion. Hey, it works!
431n/a else:
432n/a from py_compile import compile
433n/a
434n/a for file in py_files:
435n/a if file[-3:] != ".py":
436n/a # This lets us be lazy and not filter filenames in
437n/a # the "install_lib" command.
438n/a continue
439n/a
440n/a # Terminology from the py_compile module:
441n/a # cfile - byte-compiled file
442n/a # dfile - purported source filename (same as 'file' by default)
443n/a if optimize >= 0:
444n/a opt = '' if optimize == 0 else optimize
445n/a cfile = importlib.util.cache_from_source(
446n/a file, optimization=opt)
447n/a else:
448n/a cfile = importlib.util.cache_from_source(file)
449n/a dfile = file
450n/a if prefix:
451n/a if file[:len(prefix)] != prefix:
452n/a raise ValueError("invalid prefix: filename %r doesn't start with %r"
453n/a % (file, prefix))
454n/a dfile = dfile[len(prefix):]
455n/a if base_dir:
456n/a dfile = os.path.join(base_dir, dfile)
457n/a
458n/a cfile_base = os.path.basename(cfile)
459n/a if direct:
460n/a if force or newer(file, cfile):
461n/a log.info("byte-compiling %s to %s", file, cfile_base)
462n/a if not dry_run:
463n/a compile(file, cfile, dfile)
464n/a else:
465n/a log.debug("skipping byte-compilation of %s to %s",
466n/a file, cfile_base)
467n/a
468n/a# byte_compile ()
469n/a
470n/adef rfc822_escape (header):
471n/a """Return a version of the string escaped for inclusion in an
472n/a RFC-822 header, by ensuring there are 8 spaces space after each newline.
473n/a """
474n/a lines = header.split('\n')
475n/a sep = '\n' + 8 * ' '
476n/a return sep.join(lines)
477n/a
478n/a# 2to3 support
479n/a
480n/adef run_2to3(files, fixer_names=None, options=None, explicit=None):
481n/a """Invoke 2to3 on a list of Python files.
482n/a The files should all come from the build area, as the
483n/a modification is done in-place. To reduce the build time,
484n/a only files modified since the last invocation of this
485n/a function should be passed in the files argument."""
486n/a
487n/a if not files:
488n/a return
489n/a
490n/a # Make this class local, to delay import of 2to3
491n/a from lib2to3.refactor import RefactoringTool, get_fixers_from_package
492n/a class DistutilsRefactoringTool(RefactoringTool):
493n/a def log_error(self, msg, *args, **kw):
494n/a log.error(msg, *args)
495n/a
496n/a def log_message(self, msg, *args):
497n/a log.info(msg, *args)
498n/a
499n/a def log_debug(self, msg, *args):
500n/a log.debug(msg, *args)
501n/a
502n/a if fixer_names is None:
503n/a fixer_names = get_fixers_from_package('lib2to3.fixes')
504n/a r = DistutilsRefactoringTool(fixer_names, options=options)
505n/a r.refactor(files, write=True)
506n/a
507n/adef copydir_run_2to3(src, dest, template=None, fixer_names=None,
508n/a options=None, explicit=None):
509n/a """Recursively copy a directory, only copying new and changed files,
510n/a running run_2to3 over all newly copied Python modules afterward.
511n/a
512n/a If you give a template string, it's parsed like a MANIFEST.in.
513n/a """
514n/a from distutils.dir_util import mkpath
515n/a from distutils.file_util import copy_file
516n/a from distutils.filelist import FileList
517n/a filelist = FileList()
518n/a curdir = os.getcwd()
519n/a os.chdir(src)
520n/a try:
521n/a filelist.findall()
522n/a finally:
523n/a os.chdir(curdir)
524n/a filelist.files[:] = filelist.allfiles
525n/a if template:
526n/a for line in template.splitlines():
527n/a line = line.strip()
528n/a if not line: continue
529n/a filelist.process_template_line(line)
530n/a copied = []
531n/a for filename in filelist.files:
532n/a outname = os.path.join(dest, filename)
533n/a mkpath(os.path.dirname(outname))
534n/a res = copy_file(os.path.join(src, filename), outname, update=1)
535n/a if res[1]: copied.append(outname)
536n/a run_2to3([fn for fn in copied if fn.lower().endswith('.py')],
537n/a fixer_names=fixer_names, options=options, explicit=explicit)
538n/a return copied
539n/a
540n/aclass Mixin2to3:
541n/a '''Mixin class for commands that run 2to3.
542n/a To configure 2to3, setup scripts may either change
543n/a the class variables, or inherit from individual commands
544n/a to override how 2to3 is invoked.'''
545n/a
546n/a # provide list of fixers to run;
547n/a # defaults to all from lib2to3.fixers
548n/a fixer_names = None
549n/a
550n/a # options dictionary
551n/a options = None
552n/a
553n/a # list of fixers to invoke even though they are marked as explicit
554n/a explicit = None
555n/a
556n/a def run_2to3(self, files):
557n/a return run_2to3(files, self.fixer_names, self.options, self.explicit)