ยปCore Development>Code coverage>Lib/pkgutil.py

Python code coverage for Lib/pkgutil.py

#countcontent
1n/a"""Utilities to support packages."""
2n/a
3n/afrom collections import namedtuple
4n/afrom functools import singledispatch as simplegeneric
5n/aimport importlib
6n/aimport importlib.util
7n/aimport importlib.machinery
8n/aimport os
9n/aimport os.path
10n/aimport sys
11n/afrom types import ModuleType
12n/aimport warnings
13n/a
14n/a__all__ = [
15n/a 'get_importer', 'iter_importers', 'get_loader', 'find_loader',
16n/a 'walk_packages', 'iter_modules', 'get_data',
17n/a 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
18n/a 'ModuleInfo',
19n/a]
20n/a
21n/a
22n/aModuleInfo = namedtuple('ModuleInfo', 'module_finder name ispkg')
23n/aModuleInfo.__doc__ = 'A namedtuple with minimal info about a module.'
24n/a
25n/a
26n/adef _get_spec(finder, name):
27n/a """Return the finder-specific module spec."""
28n/a # Works with legacy finders.
29n/a try:
30n/a find_spec = finder.find_spec
31n/a except AttributeError:
32n/a loader = finder.find_module(name)
33n/a if loader is None:
34n/a return None
35n/a return importlib.util.spec_from_loader(name, loader)
36n/a else:
37n/a return find_spec(name)
38n/a
39n/a
40n/adef read_code(stream):
41n/a # This helper is needed in order for the PEP 302 emulation to
42n/a # correctly handle compiled files
43n/a import marshal
44n/a
45n/a magic = stream.read(4)
46n/a if magic != importlib.util.MAGIC_NUMBER:
47n/a return None
48n/a
49n/a stream.read(8) # Skip timestamp and size
50n/a return marshal.load(stream)
51n/a
52n/a
53n/adef walk_packages(path=None, prefix='', onerror=None):
54n/a """Yields ModuleInfo for all modules recursively
55n/a on path, or, if path is None, all accessible modules.
56n/a
57n/a 'path' should be either None or a list of paths to look for
58n/a modules in.
59n/a
60n/a 'prefix' is a string to output on the front of every module name
61n/a on output.
62n/a
63n/a Note that this function must import all *packages* (NOT all
64n/a modules!) on the given path, in order to access the __path__
65n/a attribute to find submodules.
66n/a
67n/a 'onerror' is a function which gets called with one argument (the
68n/a name of the package which was being imported) if any exception
69n/a occurs while trying to import a package. If no onerror function is
70n/a supplied, ImportErrors are caught and ignored, while all other
71n/a exceptions are propagated, terminating the search.
72n/a
73n/a Examples:
74n/a
75n/a # list all modules python can access
76n/a walk_packages()
77n/a
78n/a # list all submodules of ctypes
79n/a walk_packages(ctypes.__path__, ctypes.__name__+'.')
80n/a """
81n/a
82n/a def seen(p, m={}):
83n/a if p in m:
84n/a return True
85n/a m[p] = True
86n/a
87n/a for info in iter_modules(path, prefix):
88n/a yield info
89n/a
90n/a if info.ispkg:
91n/a try:
92n/a __import__(info.name)
93n/a except ImportError:
94n/a if onerror is not None:
95n/a onerror(info.name)
96n/a except Exception:
97n/a if onerror is not None:
98n/a onerror(info.name)
99n/a else:
100n/a raise
101n/a else:
102n/a path = getattr(sys.modules[info.name], '__path__', None) or []
103n/a
104n/a # don't traverse path items we've seen before
105n/a path = [p for p in path if not seen(p)]
106n/a
107n/a yield from walk_packages(path, info.name+'.', onerror)
108n/a
109n/a
110n/adef iter_modules(path=None, prefix=''):
111n/a """Yields ModuleInfo for all submodules on path,
112n/a or, if path is None, all top-level modules on sys.path.
113n/a
114n/a 'path' should be either None or a list of paths to look for
115n/a modules in.
116n/a
117n/a 'prefix' is a string to output on the front of every module name
118n/a on output.
119n/a """
120n/a if path is None:
121n/a importers = iter_importers()
122n/a else:
123n/a importers = map(get_importer, path)
124n/a
125n/a yielded = {}
126n/a for i in importers:
127n/a for name, ispkg in iter_importer_modules(i, prefix):
128n/a if name not in yielded:
129n/a yielded[name] = 1
130n/a yield ModuleInfo(i, name, ispkg)
131n/a
132n/a
133n/a@simplegeneric
134n/adef iter_importer_modules(importer, prefix=''):
135n/a if not hasattr(importer, 'iter_modules'):
136n/a return []
137n/a return importer.iter_modules(prefix)
138n/a
139n/a
140n/a# Implement a file walker for the normal importlib path hook
141n/adef _iter_file_finder_modules(importer, prefix=''):
142n/a if importer.path is None or not os.path.isdir(importer.path):
143n/a return
144n/a
145n/a yielded = {}
146n/a import inspect
147n/a try:
148n/a filenames = os.listdir(importer.path)
149n/a except OSError:
150n/a # ignore unreadable directories like import does
151n/a filenames = []
152n/a filenames.sort() # handle packages before same-named modules
153n/a
154n/a for fn in filenames:
155n/a modname = inspect.getmodulename(fn)
156n/a if modname=='__init__' or modname in yielded:
157n/a continue
158n/a
159n/a path = os.path.join(importer.path, fn)
160n/a ispkg = False
161n/a
162n/a if not modname and os.path.isdir(path) and '.' not in fn:
163n/a modname = fn
164n/a try:
165n/a dircontents = os.listdir(path)
166n/a except OSError:
167n/a # ignore unreadable directories like import does
168n/a dircontents = []
169n/a for fn in dircontents:
170n/a subname = inspect.getmodulename(fn)
171n/a if subname=='__init__':
172n/a ispkg = True
173n/a break
174n/a else:
175n/a continue # not a package
176n/a
177n/a if modname and '.' not in modname:
178n/a yielded[modname] = 1
179n/a yield prefix + modname, ispkg
180n/a
181n/aiter_importer_modules.register(
182n/a importlib.machinery.FileFinder, _iter_file_finder_modules)
183n/a
184n/a
185n/adef _import_imp():
186n/a global imp
187n/a with warnings.catch_warnings():
188n/a warnings.simplefilter('ignore', DeprecationWarning)
189n/a imp = importlib.import_module('imp')
190n/a
191n/aclass ImpImporter:
192n/a """PEP 302 Finder that wraps Python's "classic" import algorithm
193n/a
194n/a ImpImporter(dirname) produces a PEP 302 finder that searches that
195n/a directory. ImpImporter(None) produces a PEP 302 finder that searches
196n/a the current sys.path, plus any modules that are frozen or built-in.
197n/a
198n/a Note that ImpImporter does not currently support being used by placement
199n/a on sys.meta_path.
200n/a """
201n/a
202n/a def __init__(self, path=None):
203n/a global imp
204n/a warnings.warn("This emulation is deprecated, use 'importlib' instead",
205n/a DeprecationWarning)
206n/a _import_imp()
207n/a self.path = path
208n/a
209n/a def find_module(self, fullname, path=None):
210n/a # Note: we ignore 'path' argument since it is only used via meta_path
211n/a subname = fullname.split(".")[-1]
212n/a if subname != fullname and self.path is None:
213n/a return None
214n/a if self.path is None:
215n/a path = None
216n/a else:
217n/a path = [os.path.realpath(self.path)]
218n/a try:
219n/a file, filename, etc = imp.find_module(subname, path)
220n/a except ImportError:
221n/a return None
222n/a return ImpLoader(fullname, file, filename, etc)
223n/a
224n/a def iter_modules(self, prefix=''):
225n/a if self.path is None or not os.path.isdir(self.path):
226n/a return
227n/a
228n/a yielded = {}
229n/a import inspect
230n/a try:
231n/a filenames = os.listdir(self.path)
232n/a except OSError:
233n/a # ignore unreadable directories like import does
234n/a filenames = []
235n/a filenames.sort() # handle packages before same-named modules
236n/a
237n/a for fn in filenames:
238n/a modname = inspect.getmodulename(fn)
239n/a if modname=='__init__' or modname in yielded:
240n/a continue
241n/a
242n/a path = os.path.join(self.path, fn)
243n/a ispkg = False
244n/a
245n/a if not modname and os.path.isdir(path) and '.' not in fn:
246n/a modname = fn
247n/a try:
248n/a dircontents = os.listdir(path)
249n/a except OSError:
250n/a # ignore unreadable directories like import does
251n/a dircontents = []
252n/a for fn in dircontents:
253n/a subname = inspect.getmodulename(fn)
254n/a if subname=='__init__':
255n/a ispkg = True
256n/a break
257n/a else:
258n/a continue # not a package
259n/a
260n/a if modname and '.' not in modname:
261n/a yielded[modname] = 1
262n/a yield prefix + modname, ispkg
263n/a
264n/a
265n/aclass ImpLoader:
266n/a """PEP 302 Loader that wraps Python's "classic" import algorithm
267n/a """
268n/a code = source = None
269n/a
270n/a def __init__(self, fullname, file, filename, etc):
271n/a warnings.warn("This emulation is deprecated, use 'importlib' instead",
272n/a DeprecationWarning)
273n/a _import_imp()
274n/a self.file = file
275n/a self.filename = filename
276n/a self.fullname = fullname
277n/a self.etc = etc
278n/a
279n/a def load_module(self, fullname):
280n/a self._reopen()
281n/a try:
282n/a mod = imp.load_module(fullname, self.file, self.filename, self.etc)
283n/a finally:
284n/a if self.file:
285n/a self.file.close()
286n/a # Note: we don't set __loader__ because we want the module to look
287n/a # normal; i.e. this is just a wrapper for standard import machinery
288n/a return mod
289n/a
290n/a def get_data(self, pathname):
291n/a with open(pathname, "rb") as file:
292n/a return file.read()
293n/a
294n/a def _reopen(self):
295n/a if self.file and self.file.closed:
296n/a mod_type = self.etc[2]
297n/a if mod_type==imp.PY_SOURCE:
298n/a self.file = open(self.filename, 'r')
299n/a elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
300n/a self.file = open(self.filename, 'rb')
301n/a
302n/a def _fix_name(self, fullname):
303n/a if fullname is None:
304n/a fullname = self.fullname
305n/a elif fullname != self.fullname:
306n/a raise ImportError("Loader for module %s cannot handle "
307n/a "module %s" % (self.fullname, fullname))
308n/a return fullname
309n/a
310n/a def is_package(self, fullname):
311n/a fullname = self._fix_name(fullname)
312n/a return self.etc[2]==imp.PKG_DIRECTORY
313n/a
314n/a def get_code(self, fullname=None):
315n/a fullname = self._fix_name(fullname)
316n/a if self.code is None:
317n/a mod_type = self.etc[2]
318n/a if mod_type==imp.PY_SOURCE:
319n/a source = self.get_source(fullname)
320n/a self.code = compile(source, self.filename, 'exec')
321n/a elif mod_type==imp.PY_COMPILED:
322n/a self._reopen()
323n/a try:
324n/a self.code = read_code(self.file)
325n/a finally:
326n/a self.file.close()
327n/a elif mod_type==imp.PKG_DIRECTORY:
328n/a self.code = self._get_delegate().get_code()
329n/a return self.code
330n/a
331n/a def get_source(self, fullname=None):
332n/a fullname = self._fix_name(fullname)
333n/a if self.source is None:
334n/a mod_type = self.etc[2]
335n/a if mod_type==imp.PY_SOURCE:
336n/a self._reopen()
337n/a try:
338n/a self.source = self.file.read()
339n/a finally:
340n/a self.file.close()
341n/a elif mod_type==imp.PY_COMPILED:
342n/a if os.path.exists(self.filename[:-1]):
343n/a with open(self.filename[:-1], 'r') as f:
344n/a self.source = f.read()
345n/a elif mod_type==imp.PKG_DIRECTORY:
346n/a self.source = self._get_delegate().get_source()
347n/a return self.source
348n/a
349n/a def _get_delegate(self):
350n/a finder = ImpImporter(self.filename)
351n/a spec = _get_spec(finder, '__init__')
352n/a return spec.loader
353n/a
354n/a def get_filename(self, fullname=None):
355n/a fullname = self._fix_name(fullname)
356n/a mod_type = self.etc[2]
357n/a if mod_type==imp.PKG_DIRECTORY:
358n/a return self._get_delegate().get_filename()
359n/a elif mod_type in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
360n/a return self.filename
361n/a return None
362n/a
363n/a
364n/atry:
365n/a import zipimport
366n/a from zipimport import zipimporter
367n/a
368n/a def iter_zipimport_modules(importer, prefix=''):
369n/a dirlist = sorted(zipimport._zip_directory_cache[importer.archive])
370n/a _prefix = importer.prefix
371n/a plen = len(_prefix)
372n/a yielded = {}
373n/a import inspect
374n/a for fn in dirlist:
375n/a if not fn.startswith(_prefix):
376n/a continue
377n/a
378n/a fn = fn[plen:].split(os.sep)
379n/a
380n/a if len(fn)==2 and fn[1].startswith('__init__.py'):
381n/a if fn[0] not in yielded:
382n/a yielded[fn[0]] = 1
383n/a yield prefix + fn[0], True
384n/a
385n/a if len(fn)!=1:
386n/a continue
387n/a
388n/a modname = inspect.getmodulename(fn[0])
389n/a if modname=='__init__':
390n/a continue
391n/a
392n/a if modname and '.' not in modname and modname not in yielded:
393n/a yielded[modname] = 1
394n/a yield prefix + modname, False
395n/a
396n/a iter_importer_modules.register(zipimporter, iter_zipimport_modules)
397n/a
398n/aexcept ImportError:
399n/a pass
400n/a
401n/a
402n/adef get_importer(path_item):
403n/a """Retrieve a finder for the given path item
404n/a
405n/a The returned finder is cached in sys.path_importer_cache
406n/a if it was newly created by a path hook.
407n/a
408n/a The cache (or part of it) can be cleared manually if a
409n/a rescan of sys.path_hooks is necessary.
410n/a """
411n/a try:
412n/a importer = sys.path_importer_cache[path_item]
413n/a except KeyError:
414n/a for path_hook in sys.path_hooks:
415n/a try:
416n/a importer = path_hook(path_item)
417n/a sys.path_importer_cache.setdefault(path_item, importer)
418n/a break
419n/a except ImportError:
420n/a pass
421n/a else:
422n/a importer = None
423n/a return importer
424n/a
425n/a
426n/adef iter_importers(fullname=""):
427n/a """Yield finders for the given module name
428n/a
429n/a If fullname contains a '.', the finders will be for the package
430n/a containing fullname, otherwise they will be all registered top level
431n/a finders (i.e. those on both sys.meta_path and sys.path_hooks).
432n/a
433n/a If the named module is in a package, that package is imported as a side
434n/a effect of invoking this function.
435n/a
436n/a If no module name is specified, all top level finders are produced.
437n/a """
438n/a if fullname.startswith('.'):
439n/a msg = "Relative module name {!r} not supported".format(fullname)
440n/a raise ImportError(msg)
441n/a if '.' in fullname:
442n/a # Get the containing package's __path__
443n/a pkg_name = fullname.rpartition(".")[0]
444n/a pkg = importlib.import_module(pkg_name)
445n/a path = getattr(pkg, '__path__', None)
446n/a if path is None:
447n/a return
448n/a else:
449n/a yield from sys.meta_path
450n/a path = sys.path
451n/a for item in path:
452n/a yield get_importer(item)
453n/a
454n/a
455n/adef get_loader(module_or_name):
456n/a """Get a "loader" object for module_or_name
457n/a
458n/a Returns None if the module cannot be found or imported.
459n/a If the named module is not already imported, its containing package
460n/a (if any) is imported, in order to establish the package __path__.
461n/a """
462n/a if module_or_name in sys.modules:
463n/a module_or_name = sys.modules[module_or_name]
464n/a if module_or_name is None:
465n/a return None
466n/a if isinstance(module_or_name, ModuleType):
467n/a module = module_or_name
468n/a loader = getattr(module, '__loader__', None)
469n/a if loader is not None:
470n/a return loader
471n/a if getattr(module, '__spec__', None) is None:
472n/a return None
473n/a fullname = module.__name__
474n/a else:
475n/a fullname = module_or_name
476n/a return find_loader(fullname)
477n/a
478n/a
479n/adef find_loader(fullname):
480n/a """Find a "loader" object for fullname
481n/a
482n/a This is a backwards compatibility wrapper around
483n/a importlib.util.find_spec that converts most failures to ImportError
484n/a and only returns the loader rather than the full spec
485n/a """
486n/a if fullname.startswith('.'):
487n/a msg = "Relative module name {!r} not supported".format(fullname)
488n/a raise ImportError(msg)
489n/a try:
490n/a spec = importlib.util.find_spec(fullname)
491n/a except (ImportError, AttributeError, TypeError, ValueError) as ex:
492n/a # This hack fixes an impedance mismatch between pkgutil and
493n/a # importlib, where the latter raises other errors for cases where
494n/a # pkgutil previously raised ImportError
495n/a msg = "Error while finding loader for {!r} ({}: {})"
496n/a raise ImportError(msg.format(fullname, type(ex), ex)) from ex
497n/a return spec.loader if spec is not None else None
498n/a
499n/a
500n/adef extend_path(path, name):
501n/a """Extend a package's path.
502n/a
503n/a Intended use is to place the following code in a package's __init__.py:
504n/a
505n/a from pkgutil import extend_path
506n/a __path__ = extend_path(__path__, __name__)
507n/a
508n/a This will add to the package's __path__ all subdirectories of
509n/a directories on sys.path named after the package. This is useful
510n/a if one wants to distribute different parts of a single logical
511n/a package as multiple directories.
512n/a
513n/a It also looks for *.pkg files beginning where * matches the name
514n/a argument. This feature is similar to *.pth files (see site.py),
515n/a except that it doesn't special-case lines starting with 'import'.
516n/a A *.pkg file is trusted at face value: apart from checking for
517n/a duplicates, all entries found in a *.pkg file are added to the
518n/a path, regardless of whether they are exist the filesystem. (This
519n/a is a feature.)
520n/a
521n/a If the input path is not a list (as is the case for frozen
522n/a packages) it is returned unchanged. The input path is not
523n/a modified; an extended copy is returned. Items are only appended
524n/a to the copy at the end.
525n/a
526n/a It is assumed that sys.path is a sequence. Items of sys.path that
527n/a are not (unicode or 8-bit) strings referring to existing
528n/a directories are ignored. Unicode items of sys.path that cause
529n/a errors when used as filenames may cause this function to raise an
530n/a exception (in line with os.path.isdir() behavior).
531n/a """
532n/a
533n/a if not isinstance(path, list):
534n/a # This could happen e.g. when this is called from inside a
535n/a # frozen package. Return the path unchanged in that case.
536n/a return path
537n/a
538n/a sname_pkg = name + ".pkg"
539n/a
540n/a path = path[:] # Start with a copy of the existing path
541n/a
542n/a parent_package, _, final_name = name.rpartition('.')
543n/a if parent_package:
544n/a try:
545n/a search_path = sys.modules[parent_package].__path__
546n/a except (KeyError, AttributeError):
547n/a # We can't do anything: find_loader() returns None when
548n/a # passed a dotted name.
549n/a return path
550n/a else:
551n/a search_path = sys.path
552n/a
553n/a for dir in search_path:
554n/a if not isinstance(dir, str):
555n/a continue
556n/a
557n/a finder = get_importer(dir)
558n/a if finder is not None:
559n/a portions = []
560n/a if hasattr(finder, 'find_spec'):
561n/a spec = finder.find_spec(final_name)
562n/a if spec is not None:
563n/a portions = spec.submodule_search_locations or []
564n/a # Is this finder PEP 420 compliant?
565n/a elif hasattr(finder, 'find_loader'):
566n/a _, portions = finder.find_loader(final_name)
567n/a
568n/a for portion in portions:
569n/a # XXX This may still add duplicate entries to path on
570n/a # case-insensitive filesystems
571n/a if portion not in path:
572n/a path.append(portion)
573n/a
574n/a # XXX Is this the right thing for subpackages like zope.app?
575n/a # It looks for a file named "zope.app.pkg"
576n/a pkgfile = os.path.join(dir, sname_pkg)
577n/a if os.path.isfile(pkgfile):
578n/a try:
579n/a f = open(pkgfile)
580n/a except OSError as msg:
581n/a sys.stderr.write("Can't open %s: %s\n" %
582n/a (pkgfile, msg))
583n/a else:
584n/a with f:
585n/a for line in f:
586n/a line = line.rstrip('\n')
587n/a if not line or line.startswith('#'):
588n/a continue
589n/a path.append(line) # Don't check for existence!
590n/a
591n/a return path
592n/a
593n/a
594n/adef get_data(package, resource):
595n/a """Get a resource from a package.
596n/a
597n/a This is a wrapper round the PEP 302 loader get_data API. The package
598n/a argument should be the name of a package, in standard module format
599n/a (foo.bar). The resource argument should be in the form of a relative
600n/a filename, using '/' as the path separator. The parent directory name '..'
601n/a is not allowed, and nor is a rooted name (starting with a '/').
602n/a
603n/a The function returns a binary string, which is the contents of the
604n/a specified resource.
605n/a
606n/a For packages located in the filesystem, which have already been imported,
607n/a this is the rough equivalent of
608n/a
609n/a d = os.path.dirname(sys.modules[package].__file__)
610n/a data = open(os.path.join(d, resource), 'rb').read()
611n/a
612n/a If the package cannot be located or loaded, or it uses a PEP 302 loader
613n/a which does not support get_data(), then None is returned.
614n/a """
615n/a
616n/a spec = importlib.util.find_spec(package)
617n/a if spec is None:
618n/a return None
619n/a loader = spec.loader
620n/a if loader is None or not hasattr(loader, 'get_data'):
621n/a return None
622n/a # XXX needs test
623n/a mod = (sys.modules.get(package) or
624n/a importlib._bootstrap._load(spec))
625n/a if mod is None or not hasattr(mod, '__file__'):
626n/a return None
627n/a
628n/a # Modify the resource name to be compatible with the loader.get_data
629n/a # signature - an os.path format "filename" starting with the dirname of
630n/a # the package's __file__
631n/a parts = resource.split('/')
632n/a parts.insert(0, os.path.dirname(mod.__file__))
633n/a resource_name = os.path.join(*parts)
634n/a return loader.get_data(resource_name)