ยปCore Development>Code coverage>Lib/packaging/database.py

Python code coverage for Lib/packaging/database.py

#countcontent
1n/a"""PEP 376 implementation."""
2n/a
3n/aimport os
4n/aimport re
5n/aimport csv
6n/aimport sys
7n/aimport zipimport
8n/afrom io import StringIO
9n/afrom hashlib import md5
10n/a
11n/afrom packaging import logger
12n/afrom packaging.errors import PackagingError
13n/afrom packaging.version import suggest_normalized_version, VersionPredicate
14n/afrom packaging.metadata import Metadata
15n/a
16n/a
17n/a__all__ = [
18n/a 'Distribution', 'EggInfoDistribution', 'distinfo_dirname',
19n/a 'get_distributions', 'get_distribution', 'get_file_users',
20n/a 'provides_distribution', 'obsoletes_distribution',
21n/a 'enable_cache', 'disable_cache', 'clear_cache',
22n/a # XXX these functions' names look like get_file_users but are not related
23n/a 'get_file_path', 'get_file']
24n/a
25n/a
26n/a# TODO update docs
27n/a
28n/aDIST_FILES = ('INSTALLER', 'METADATA', 'RECORD', 'REQUESTED', 'RESOURCES')
29n/a
30n/a# Cache
31n/a_cache_name = {} # maps names to Distribution instances
32n/a_cache_name_egg = {} # maps names to EggInfoDistribution instances
33n/a_cache_path = {} # maps paths to Distribution instances
34n/a_cache_path_egg = {} # maps paths to EggInfoDistribution instances
35n/a_cache_generated = False # indicates if .dist-info distributions are cached
36n/a_cache_generated_egg = False # indicates if .dist-info and .egg are cached
37n/a_cache_enabled = True
38n/a
39n/a
40n/adef enable_cache():
41n/a """
42n/a Enables the internal cache.
43n/a
44n/a Note that this function will not clear the cache in any case, for that
45n/a functionality see :func:`clear_cache`.
46n/a """
47n/a global _cache_enabled
48n/a
49n/a _cache_enabled = True
50n/a
51n/a
52n/adef disable_cache():
53n/a """
54n/a Disables the internal cache.
55n/a
56n/a Note that this function will not clear the cache in any case, for that
57n/a functionality see :func:`clear_cache`.
58n/a """
59n/a global _cache_enabled
60n/a
61n/a _cache_enabled = False
62n/a
63n/a
64n/adef clear_cache():
65n/a """ Clears the internal cache. """
66n/a global _cache_generated, _cache_generated_egg
67n/a
68n/a _cache_name.clear()
69n/a _cache_name_egg.clear()
70n/a _cache_path.clear()
71n/a _cache_path_egg.clear()
72n/a _cache_generated = False
73n/a _cache_generated_egg = False
74n/a
75n/a
76n/adef _yield_distributions(include_dist, include_egg, paths):
77n/a """
78n/a Yield .dist-info and .egg(-info) distributions, based on the arguments
79n/a
80n/a :parameter include_dist: yield .dist-info distributions
81n/a :parameter include_egg: yield .egg(-info) distributions
82n/a """
83n/a for path in paths:
84n/a realpath = os.path.realpath(path)
85n/a if not os.path.isdir(realpath):
86n/a continue
87n/a for dir in os.listdir(realpath):
88n/a dist_path = os.path.join(realpath, dir)
89n/a if include_dist and dir.endswith('.dist-info'):
90n/a yield Distribution(dist_path)
91n/a elif include_egg and (dir.endswith('.egg-info') or
92n/a dir.endswith('.egg')):
93n/a yield EggInfoDistribution(dist_path)
94n/a
95n/a
96n/adef _generate_cache(use_egg_info, paths):
97n/a global _cache_generated, _cache_generated_egg
98n/a
99n/a if _cache_generated_egg or (_cache_generated and not use_egg_info):
100n/a return
101n/a else:
102n/a gen_dist = not _cache_generated
103n/a gen_egg = use_egg_info
104n/a
105n/a for dist in _yield_distributions(gen_dist, gen_egg, paths):
106n/a if isinstance(dist, Distribution):
107n/a _cache_path[dist.path] = dist
108n/a if dist.name not in _cache_name:
109n/a _cache_name[dist.name] = []
110n/a _cache_name[dist.name].append(dist)
111n/a else:
112n/a _cache_path_egg[dist.path] = dist
113n/a if dist.name not in _cache_name_egg:
114n/a _cache_name_egg[dist.name] = []
115n/a _cache_name_egg[dist.name].append(dist)
116n/a
117n/a if gen_dist:
118n/a _cache_generated = True
119n/a if gen_egg:
120n/a _cache_generated_egg = True
121n/a
122n/a
123n/aclass Distribution:
124n/a """Created with the *path* of the ``.dist-info`` directory provided to the
125n/a constructor. It reads the metadata contained in ``METADATA`` when it is
126n/a instantiated."""
127n/a
128n/a name = ''
129n/a """The name of the distribution."""
130n/a
131n/a version = ''
132n/a """The version of the distribution."""
133n/a
134n/a metadata = None
135n/a """A :class:`packaging.metadata.Metadata` instance loaded with
136n/a the distribution's ``METADATA`` file."""
137n/a
138n/a requested = False
139n/a """A boolean that indicates whether the ``REQUESTED`` metadata file is
140n/a present (in other words, whether the package was installed by user
141n/a request or it was installed as a dependency)."""
142n/a
143n/a def __init__(self, path):
144n/a if _cache_enabled and path in _cache_path:
145n/a self.metadata = _cache_path[path].metadata
146n/a else:
147n/a metadata_path = os.path.join(path, 'METADATA')
148n/a self.metadata = Metadata(path=metadata_path)
149n/a
150n/a self.name = self.metadata['Name']
151n/a self.version = self.metadata['Version']
152n/a self.path = path
153n/a
154n/a if _cache_enabled and path not in _cache_path:
155n/a _cache_path[path] = self
156n/a
157n/a def __repr__(self):
158n/a return '<Distribution %r %s at %r>' % (
159n/a self.name, self.version, self.path)
160n/a
161n/a def _get_records(self, local=False):
162n/a results = []
163n/a with self.get_distinfo_file('RECORD') as record:
164n/a record_reader = csv.reader(record, delimiter=',',
165n/a lineterminator='\n')
166n/a for row in record_reader:
167n/a missing = [None for i in range(len(row), 3)]
168n/a path, checksum, size = row + missing
169n/a if local:
170n/a path = path.replace('/', os.sep)
171n/a path = os.path.join(sys.prefix, path)
172n/a results.append((path, checksum, size))
173n/a return results
174n/a
175n/a def get_resource_path(self, relative_path):
176n/a with self.get_distinfo_file('RESOURCES') as resources_file:
177n/a resources_reader = csv.reader(resources_file, delimiter=',',
178n/a lineterminator='\n')
179n/a for relative, destination in resources_reader:
180n/a if relative == relative_path:
181n/a return destination
182n/a raise KeyError(
183n/a 'no resource file with relative path %r is installed' %
184n/a relative_path)
185n/a
186n/a def list_installed_files(self, local=False):
187n/a """
188n/a Iterates over the ``RECORD`` entries and returns a tuple
189n/a ``(path, md5, size)`` for each line. If *local* is ``True``,
190n/a the returned path is transformed into a local absolute path.
191n/a Otherwise the raw value from RECORD is returned.
192n/a
193n/a A local absolute path is an absolute path in which occurrences of
194n/a ``'/'`` have been replaced by the system separator given by ``os.sep``.
195n/a
196n/a :parameter local: flag to say if the path should be returned as a local
197n/a absolute path
198n/a
199n/a :type local: boolean
200n/a :returns: iterator of (path, md5, size)
201n/a """
202n/a for result in self._get_records(local):
203n/a yield result
204n/a
205n/a def uses(self, path):
206n/a """
207n/a Returns ``True`` if path is listed in ``RECORD``. *path* can be a local
208n/a absolute path or a relative ``'/'``-separated path.
209n/a
210n/a :rtype: boolean
211n/a """
212n/a for p, checksum, size in self._get_records():
213n/a local_absolute = os.path.join(sys.prefix, p)
214n/a if path == p or path == local_absolute:
215n/a return True
216n/a return False
217n/a
218n/a def get_distinfo_file(self, path, binary=False):
219n/a """
220n/a Returns a file located under the ``.dist-info`` directory. Returns a
221n/a ``file`` instance for the file pointed by *path*.
222n/a
223n/a :parameter path: a ``'/'``-separated path relative to the
224n/a ``.dist-info`` directory or an absolute path;
225n/a If *path* is an absolute path and doesn't start
226n/a with the ``.dist-info`` directory path,
227n/a a :class:`PackagingError` is raised
228n/a :type path: string
229n/a :parameter binary: If *binary* is ``True``, opens the file in read-only
230n/a binary mode (``rb``), otherwise opens it in
231n/a read-only mode (``r``).
232n/a :rtype: file object
233n/a """
234n/a open_flags = 'r'
235n/a if binary:
236n/a open_flags += 'b'
237n/a
238n/a # Check if it is an absolute path # XXX use relpath, add tests
239n/a if path.find(os.sep) >= 0:
240n/a # it's an absolute path?
241n/a distinfo_dirname, path = path.split(os.sep)[-2:]
242n/a if distinfo_dirname != self.path.split(os.sep)[-1]:
243n/a raise PackagingError(
244n/a 'dist-info file %r does not belong to the %r %s '
245n/a 'distribution' % (path, self.name, self.version))
246n/a
247n/a # The file must be relative
248n/a if path not in DIST_FILES:
249n/a raise PackagingError('invalid path for a dist-info file: %r' %
250n/a path)
251n/a
252n/a path = os.path.join(self.path, path)
253n/a return open(path, open_flags)
254n/a
255n/a def list_distinfo_files(self, local=False):
256n/a """
257n/a Iterates over the ``RECORD`` entries and returns paths for each line if
258n/a the path is pointing to a file located in the ``.dist-info`` directory
259n/a or one of its subdirectories.
260n/a
261n/a :parameter local: If *local* is ``True``, each returned path is
262n/a transformed into a local absolute path. Otherwise the
263n/a raw value from ``RECORD`` is returned.
264n/a :type local: boolean
265n/a :returns: iterator of paths
266n/a """
267n/a for path, checksum, size in self._get_records(local):
268n/a # XXX add separator or use real relpath algo
269n/a if path.startswith(self.path):
270n/a yield path
271n/a
272n/a def __eq__(self, other):
273n/a return isinstance(other, Distribution) and self.path == other.path
274n/a
275n/a # See http://docs.python.org/reference/datamodel#object.__hash__
276n/a __hash__ = object.__hash__
277n/a
278n/a
279n/aclass EggInfoDistribution:
280n/a """Created with the *path* of the ``.egg-info`` directory or file provided
281n/a to the constructor. It reads the metadata contained in the file itself, or
282n/a if the given path happens to be a directory, the metadata is read from the
283n/a file ``PKG-INFO`` under that directory."""
284n/a
285n/a name = ''
286n/a """The name of the distribution."""
287n/a
288n/a version = ''
289n/a """The version of the distribution."""
290n/a
291n/a metadata = None
292n/a """A :class:`packaging.metadata.Metadata` instance loaded with
293n/a the distribution's ``METADATA`` file."""
294n/a
295n/a _REQUIREMENT = re.compile(
296n/a r'(?P<name>[-A-Za-z0-9_.]+)\s*'
297n/a r'(?P<first>(?:<|<=|!=|==|>=|>)[-A-Za-z0-9_.]+)?\s*'
298n/a r'(?P<rest>(?:\s*,\s*(?:<|<=|!=|==|>=|>)[-A-Za-z0-9_.]+)*)\s*'
299n/a r'(?P<extras>\[.*\])?')
300n/a
301n/a def __init__(self, path):
302n/a self.path = path
303n/a if _cache_enabled and path in _cache_path_egg:
304n/a self.metadata = _cache_path_egg[path].metadata
305n/a self.name = self.metadata['Name']
306n/a self.version = self.metadata['Version']
307n/a return
308n/a
309n/a # reused from Distribute's pkg_resources
310n/a def yield_lines(strs):
311n/a """Yield non-empty/non-comment lines of a ``basestring``
312n/a or sequence"""
313n/a if isinstance(strs, str):
314n/a for s in strs.splitlines():
315n/a s = s.strip()
316n/a # skip blank lines/comments
317n/a if s and not s.startswith('#'):
318n/a yield s
319n/a else:
320n/a for ss in strs:
321n/a for s in yield_lines(ss):
322n/a yield s
323n/a
324n/a requires = None
325n/a
326n/a if path.endswith('.egg'):
327n/a if os.path.isdir(path):
328n/a meta_path = os.path.join(path, 'EGG-INFO', 'PKG-INFO')
329n/a self.metadata = Metadata(path=meta_path)
330n/a try:
331n/a req_path = os.path.join(path, 'EGG-INFO', 'requires.txt')
332n/a with open(req_path, 'r') as fp:
333n/a requires = fp.read()
334n/a except IOError:
335n/a requires = None
336n/a else:
337n/a # FIXME handle the case where zipfile is not available
338n/a zipf = zipimport.zipimporter(path)
339n/a fileobj = StringIO(
340n/a zipf.get_data('EGG-INFO/PKG-INFO').decode('utf8'))
341n/a self.metadata = Metadata(fileobj=fileobj)
342n/a try:
343n/a requires = zipf.get_data('EGG-INFO/requires.txt')
344n/a except IOError:
345n/a requires = None
346n/a self.name = self.metadata['Name']
347n/a self.version = self.metadata['Version']
348n/a
349n/a elif path.endswith('.egg-info'):
350n/a if os.path.isdir(path):
351n/a path = os.path.join(path, 'PKG-INFO')
352n/a try:
353n/a with open(os.path.join(path, 'requires.txt'), 'r') as fp:
354n/a requires = fp.read()
355n/a except IOError:
356n/a requires = None
357n/a self.metadata = Metadata(path=path)
358n/a self.name = self.metadata['Name']
359n/a self.version = self.metadata['Version']
360n/a
361n/a else:
362n/a raise ValueError('path must end with .egg-info or .egg, got %r' %
363n/a path)
364n/a
365n/a if requires is not None:
366n/a if self.metadata['Metadata-Version'] == '1.1':
367n/a # we can't have 1.1 metadata *and* Setuptools requires
368n/a for field in ('Obsoletes', 'Requires', 'Provides'):
369n/a del self.metadata[field]
370n/a
371n/a reqs = []
372n/a
373n/a if requires is not None:
374n/a for line in yield_lines(requires):
375n/a if line.startswith('['):
376n/a logger.warning(
377n/a 'extensions in requires.txt are not supported '
378n/a '(used by %r %s)', self.name, self.version)
379n/a break
380n/a else:
381n/a match = self._REQUIREMENT.match(line.strip())
382n/a if not match:
383n/a # this happens when we encounter extras; since they
384n/a # are written at the end of the file we just exit
385n/a break
386n/a else:
387n/a if match.group('extras'):
388n/a msg = ('extra requirements are not supported '
389n/a '(used by %r %s)', self.name, self.version)
390n/a logger.warning(msg, self.name)
391n/a name = match.group('name')
392n/a version = None
393n/a if match.group('first'):
394n/a version = match.group('first')
395n/a if match.group('rest'):
396n/a version += match.group('rest')
397n/a version = version.replace(' ', '') # trim spaces
398n/a if version is None:
399n/a reqs.append(name)
400n/a else:
401n/a reqs.append('%s (%s)' % (name, version))
402n/a
403n/a if len(reqs) > 0:
404n/a self.metadata['Requires-Dist'] += reqs
405n/a
406n/a if _cache_enabled:
407n/a _cache_path_egg[self.path] = self
408n/a
409n/a def __repr__(self):
410n/a return '<EggInfoDistribution %r %s at %r>' % (
411n/a self.name, self.version, self.path)
412n/a
413n/a def list_installed_files(self, local=False):
414n/a
415n/a def _md5(path):
416n/a with open(path, 'rb') as f:
417n/a content = f.read()
418n/a return md5(content).hexdigest()
419n/a
420n/a def _size(path):
421n/a return os.stat(path).st_size
422n/a
423n/a path = self.path
424n/a if local:
425n/a path = path.replace('/', os.sep)
426n/a
427n/a # XXX What about scripts and data files ?
428n/a if os.path.isfile(path):
429n/a return [(path, _md5(path), _size(path))]
430n/a else:
431n/a files = []
432n/a for root, dir, files_ in os.walk(path):
433n/a for item in files_:
434n/a item = os.path.join(root, item)
435n/a files.append((item, _md5(item), _size(item)))
436n/a return files
437n/a
438n/a return []
439n/a
440n/a def uses(self, path):
441n/a return False
442n/a
443n/a def __eq__(self, other):
444n/a return (isinstance(other, EggInfoDistribution) and
445n/a self.path == other.path)
446n/a
447n/a # See http://docs.python.org/reference/datamodel#object.__hash__
448n/a __hash__ = object.__hash__
449n/a
450n/a
451n/adef distinfo_dirname(name, version):
452n/a """
453n/a The *name* and *version* parameters are converted into their
454n/a filename-escaped form, i.e. any ``'-'`` characters are replaced
455n/a with ``'_'`` other than the one in ``'dist-info'`` and the one
456n/a separating the name from the version number.
457n/a
458n/a :parameter name: is converted to a standard distribution name by replacing
459n/a any runs of non- alphanumeric characters with a single
460n/a ``'-'``.
461n/a :type name: string
462n/a :parameter version: is converted to a standard version string. Spaces
463n/a become dots, and all other non-alphanumeric characters
464n/a (except dots) become dashes, with runs of multiple
465n/a dashes condensed to a single dash.
466n/a :type version: string
467n/a :returns: directory name
468n/a :rtype: string"""
469n/a file_extension = '.dist-info'
470n/a name = name.replace('-', '_')
471n/a normalized_version = suggest_normalized_version(version)
472n/a # Because this is a lookup procedure, something will be returned even if
473n/a # it is a version that cannot be normalized
474n/a if normalized_version is None:
475n/a # Unable to achieve normality?
476n/a normalized_version = version
477n/a return '-'.join([name, normalized_version]) + file_extension
478n/a
479n/a
480n/adef get_distributions(use_egg_info=False, paths=None):
481n/a """
482n/a Provides an iterator that looks for ``.dist-info`` directories in
483n/a ``sys.path`` and returns :class:`Distribution` instances for each one of
484n/a them. If the parameters *use_egg_info* is ``True``, then the ``.egg-info``
485n/a files and directores are iterated as well.
486n/a
487n/a :rtype: iterator of :class:`Distribution` and :class:`EggInfoDistribution`
488n/a instances
489n/a """
490n/a if paths is None:
491n/a paths = sys.path
492n/a
493n/a if not _cache_enabled:
494n/a for dist in _yield_distributions(True, use_egg_info, paths):
495n/a yield dist
496n/a else:
497n/a _generate_cache(use_egg_info, paths)
498n/a
499n/a for dist in _cache_path.values():
500n/a yield dist
501n/a
502n/a if use_egg_info:
503n/a for dist in _cache_path_egg.values():
504n/a yield dist
505n/a
506n/a
507n/adef get_distribution(name, use_egg_info=False, paths=None):
508n/a """
509n/a Scans all elements in ``sys.path`` and looks for all directories
510n/a ending with ``.dist-info``. Returns a :class:`Distribution`
511n/a corresponding to the ``.dist-info`` directory that contains the
512n/a ``METADATA`` that matches *name* for the *name* metadata field.
513n/a If no distribution exists with the given *name* and the parameter
514n/a *use_egg_info* is set to ``True``, then all files and directories ending
515n/a with ``.egg-info`` are scanned. A :class:`EggInfoDistribution` instance is
516n/a returned if one is found that has metadata that matches *name* for the
517n/a *name* metadata field.
518n/a
519n/a This function only returns the first result found, as no more than one
520n/a value is expected. If the directory is not found, ``None`` is returned.
521n/a
522n/a :rtype: :class:`Distribution` or :class:`EggInfoDistribution` or None
523n/a """
524n/a if paths is None:
525n/a paths = sys.path
526n/a
527n/a if not _cache_enabled:
528n/a for dist in _yield_distributions(True, use_egg_info, paths):
529n/a if dist.name == name:
530n/a return dist
531n/a else:
532n/a _generate_cache(use_egg_info, paths)
533n/a
534n/a if name in _cache_name:
535n/a return _cache_name[name][0]
536n/a elif use_egg_info and name in _cache_name_egg:
537n/a return _cache_name_egg[name][0]
538n/a else:
539n/a return None
540n/a
541n/a
542n/adef obsoletes_distribution(name, version=None, use_egg_info=False):
543n/a """
544n/a Iterates over all distributions to find which distributions obsolete
545n/a *name*.
546n/a
547n/a If a *version* is provided, it will be used to filter the results.
548n/a If the argument *use_egg_info* is set to ``True``, then ``.egg-info``
549n/a distributions will be considered as well.
550n/a
551n/a :type name: string
552n/a :type version: string
553n/a :parameter name:
554n/a """
555n/a for dist in get_distributions(use_egg_info):
556n/a obsoleted = (dist.metadata['Obsoletes-Dist'] +
557n/a dist.metadata['Obsoletes'])
558n/a for obs in obsoleted:
559n/a o_components = obs.split(' ', 1)
560n/a if len(o_components) == 1 or version is None:
561n/a if name == o_components[0]:
562n/a yield dist
563n/a break
564n/a else:
565n/a try:
566n/a predicate = VersionPredicate(obs)
567n/a except ValueError:
568n/a raise PackagingError(
569n/a 'distribution %r has ill-formed obsoletes field: '
570n/a '%r' % (dist.name, obs))
571n/a if name == o_components[0] and predicate.match(version):
572n/a yield dist
573n/a break
574n/a
575n/a
576n/adef provides_distribution(name, version=None, use_egg_info=False):
577n/a """
578n/a Iterates over all distributions to find which distributions provide *name*.
579n/a If a *version* is provided, it will be used to filter the results. Scans
580n/a all elements in ``sys.path`` and looks for all directories ending with
581n/a ``.dist-info``. Returns a :class:`Distribution` corresponding to the
582n/a ``.dist-info`` directory that contains a ``METADATA`` that matches *name*
583n/a for the name metadata. If the argument *use_egg_info* is set to ``True``,
584n/a then all files and directories ending with ``.egg-info`` are considered
585n/a as well and returns an :class:`EggInfoDistribution` instance.
586n/a
587n/a This function only returns the first result found, since no more than
588n/a one values are expected. If the directory is not found, returns ``None``.
589n/a
590n/a :parameter version: a version specifier that indicates the version
591n/a required, conforming to the format in ``PEP-345``
592n/a
593n/a :type name: string
594n/a :type version: string
595n/a """
596n/a predicate = None
597n/a if not version is None:
598n/a try:
599n/a predicate = VersionPredicate(name + ' (' + version + ')')
600n/a except ValueError:
601n/a raise PackagingError('invalid name or version: %r, %r' %
602n/a (name, version))
603n/a
604n/a for dist in get_distributions(use_egg_info):
605n/a provided = dist.metadata['Provides-Dist'] + dist.metadata['Provides']
606n/a
607n/a for p in provided:
608n/a p_components = p.rsplit(' ', 1)
609n/a if len(p_components) == 1 or predicate is None:
610n/a if name == p_components[0]:
611n/a yield dist
612n/a break
613n/a else:
614n/a p_name, p_ver = p_components
615n/a if len(p_ver) < 2 or p_ver[0] != '(' or p_ver[-1] != ')':
616n/a raise PackagingError(
617n/a 'distribution %r has invalid Provides field: %r' %
618n/a (dist.name, p))
619n/a p_ver = p_ver[1:-1] # trim off the parenthesis
620n/a if p_name == name and predicate.match(p_ver):
621n/a yield dist
622n/a break
623n/a
624n/a
625n/adef get_file_users(path):
626n/a """
627n/a Iterates over all distributions to find out which distributions use
628n/a *path*.
629n/a
630n/a :parameter path: can be a local absolute path or a relative
631n/a ``'/'``-separated path.
632n/a :type path: string
633n/a :rtype: iterator of :class:`Distribution` instances
634n/a """
635n/a for dist in get_distributions():
636n/a if dist.uses(path):
637n/a yield dist
638n/a
639n/a
640n/adef get_file_path(distribution_name, relative_path):
641n/a """Return the path to a resource file."""
642n/a dist = get_distribution(distribution_name)
643n/a if dist is not None:
644n/a return dist.get_resource_path(relative_path)
645n/a raise LookupError('no distribution named %r found' % distribution_name)
646n/a
647n/a
648n/adef get_file(distribution_name, relative_path, *args, **kwargs):
649n/a """Open and return a resource file."""
650n/a return open(get_file_path(distribution_name, relative_path),
651n/a *args, **kwargs)