ยปCore Development>Code coverage>Lib/shelve.py

Python code coverage for Lib/shelve.py

#countcontent
1n/a"""Manage shelves of pickled objects.
2n/a
3n/aA "shelf" is a persistent, dictionary-like object. The difference
4n/awith dbm databases is that the values (not the keys!) in a shelf can
5n/abe essentially arbitrary Python objects -- anything that the "pickle"
6n/amodule can handle. This includes most class instances, recursive data
7n/atypes, and objects containing lots of shared sub-objects. The keys
8n/aare ordinary strings.
9n/a
10n/aTo summarize the interface (key is a string, data is an arbitrary
11n/aobject):
12n/a
13n/a import shelve
14n/a d = shelve.open(filename) # open, with (g)dbm filename -- no suffix
15n/a
16n/a d[key] = data # store data at key (overwrites old data if
17n/a # using an existing key)
18n/a data = d[key] # retrieve a COPY of the data at key (raise
19n/a # KeyError if no such key) -- NOTE that this
20n/a # access returns a *copy* of the entry!
21n/a del d[key] # delete data stored at key (raises KeyError
22n/a # if no such key)
23n/a flag = key in d # true if the key exists
24n/a list = d.keys() # a list of all existing keys (slow!)
25n/a
26n/a d.close() # close it
27n/a
28n/aDependent on the implementation, closing a persistent dictionary may
29n/aor may not be necessary to flush changes to disk.
30n/a
31n/aNormally, d[key] returns a COPY of the entry. This needs care when
32n/amutable entries are mutated: for example, if d[key] is a list,
33n/a d[key].append(anitem)
34n/adoes NOT modify the entry d[key] itself, as stored in the persistent
35n/amapping -- it only modifies the copy, which is then immediately
36n/adiscarded, so that the append has NO effect whatsoever. To append an
37n/aitem to d[key] in a way that will affect the persistent mapping, use:
38n/a data = d[key]
39n/a data.append(anitem)
40n/a d[key] = data
41n/a
42n/aTo avoid the problem with mutable entries, you may pass the keyword
43n/aargument writeback=True in the call to shelve.open. When you use:
44n/a d = shelve.open(filename, writeback=True)
45n/athen d keeps a cache of all entries you access, and writes them all back
46n/ato the persistent mapping when you call d.close(). This ensures that
47n/asuch usage as d[key].append(anitem) works as intended.
48n/a
49n/aHowever, using keyword argument writeback=True may consume vast amount
50n/aof memory for the cache, and it may make d.close() very slow, if you
51n/aaccess many of d's entries after opening it in this way: d has no way to
52n/acheck which of the entries you access are mutable and/or which ones you
53n/aactually mutate, so it must cache, and write back at close, all of the
54n/aentries that you access. You can call d.sync() to write back all the
55n/aentries in the cache, and empty the cache (d.sync() also synchronizes
56n/athe persistent dictionary on disk, if feasible).
57n/a"""
58n/a
59n/afrom pickle import Pickler, Unpickler
60n/afrom io import BytesIO
61n/a
62n/aimport collections
63n/a
64n/a__all__ = ["Shelf", "BsdDbShelf", "DbfilenameShelf", "open"]
65n/a
66n/aclass _ClosedDict(collections.MutableMapping):
67n/a 'Marker for a closed dict. Access attempts raise a ValueError.'
68n/a
69n/a def closed(self, *args):
70n/a raise ValueError('invalid operation on closed shelf')
71n/a __iter__ = __len__ = __getitem__ = __setitem__ = __delitem__ = keys = closed
72n/a
73n/a def __repr__(self):
74n/a return '<Closed Dictionary>'
75n/a
76n/a
77n/aclass Shelf(collections.MutableMapping):
78n/a """Base class for shelf implementations.
79n/a
80n/a This is initialized with a dictionary-like object.
81n/a See the module's __doc__ string for an overview of the interface.
82n/a """
83n/a
84n/a def __init__(self, dict, protocol=None, writeback=False,
85n/a keyencoding="utf-8"):
86n/a self.dict = dict
87n/a if protocol is None:
88n/a protocol = 3
89n/a self._protocol = protocol
90n/a self.writeback = writeback
91n/a self.cache = {}
92n/a self.keyencoding = keyencoding
93n/a
94n/a def __iter__(self):
95n/a for k in self.dict.keys():
96n/a yield k.decode(self.keyencoding)
97n/a
98n/a def __len__(self):
99n/a return len(self.dict)
100n/a
101n/a def __contains__(self, key):
102n/a return key.encode(self.keyencoding) in self.dict
103n/a
104n/a def get(self, key, default=None):
105n/a if key.encode(self.keyencoding) in self.dict:
106n/a return self[key]
107n/a return default
108n/a
109n/a def __getitem__(self, key):
110n/a try:
111n/a value = self.cache[key]
112n/a except KeyError:
113n/a f = BytesIO(self.dict[key.encode(self.keyencoding)])
114n/a value = Unpickler(f).load()
115n/a if self.writeback:
116n/a self.cache[key] = value
117n/a return value
118n/a
119n/a def __setitem__(self, key, value):
120n/a if self.writeback:
121n/a self.cache[key] = value
122n/a f = BytesIO()
123n/a p = Pickler(f, self._protocol)
124n/a p.dump(value)
125n/a self.dict[key.encode(self.keyencoding)] = f.getvalue()
126n/a
127n/a def __delitem__(self, key):
128n/a del self.dict[key.encode(self.keyencoding)]
129n/a try:
130n/a del self.cache[key]
131n/a except KeyError:
132n/a pass
133n/a
134n/a def __enter__(self):
135n/a return self
136n/a
137n/a def __exit__(self, type, value, traceback):
138n/a self.close()
139n/a
140n/a def close(self):
141n/a if self.dict is None:
142n/a return
143n/a try:
144n/a self.sync()
145n/a try:
146n/a self.dict.close()
147n/a except AttributeError:
148n/a pass
149n/a finally:
150n/a # Catch errors that may happen when close is called from __del__
151n/a # because CPython is in interpreter shutdown.
152n/a try:
153n/a self.dict = _ClosedDict()
154n/a except:
155n/a self.dict = None
156n/a
157n/a def __del__(self):
158n/a if not hasattr(self, 'writeback'):
159n/a # __init__ didn't succeed, so don't bother closing
160n/a # see http://bugs.python.org/issue1339007 for details
161n/a return
162n/a self.close()
163n/a
164n/a def sync(self):
165n/a if self.writeback and self.cache:
166n/a self.writeback = False
167n/a for key, entry in self.cache.items():
168n/a self[key] = entry
169n/a self.writeback = True
170n/a self.cache = {}
171n/a if hasattr(self.dict, 'sync'):
172n/a self.dict.sync()
173n/a
174n/a
175n/aclass BsdDbShelf(Shelf):
176n/a """Shelf implementation using the "BSD" db interface.
177n/a
178n/a This adds methods first(), next(), previous(), last() and
179n/a set_location() that have no counterpart in [g]dbm databases.
180n/a
181n/a The actual database must be opened using one of the "bsddb"
182n/a modules "open" routines (i.e. bsddb.hashopen, bsddb.btopen or
183n/a bsddb.rnopen) and passed to the constructor.
184n/a
185n/a See the module's __doc__ string for an overview of the interface.
186n/a """
187n/a
188n/a def __init__(self, dict, protocol=None, writeback=False,
189n/a keyencoding="utf-8"):
190n/a Shelf.__init__(self, dict, protocol, writeback, keyencoding)
191n/a
192n/a def set_location(self, key):
193n/a (key, value) = self.dict.set_location(key)
194n/a f = BytesIO(value)
195n/a return (key.decode(self.keyencoding), Unpickler(f).load())
196n/a
197n/a def next(self):
198n/a (key, value) = next(self.dict)
199n/a f = BytesIO(value)
200n/a return (key.decode(self.keyencoding), Unpickler(f).load())
201n/a
202n/a def previous(self):
203n/a (key, value) = self.dict.previous()
204n/a f = BytesIO(value)
205n/a return (key.decode(self.keyencoding), Unpickler(f).load())
206n/a
207n/a def first(self):
208n/a (key, value) = self.dict.first()
209n/a f = BytesIO(value)
210n/a return (key.decode(self.keyencoding), Unpickler(f).load())
211n/a
212n/a def last(self):
213n/a (key, value) = self.dict.last()
214n/a f = BytesIO(value)
215n/a return (key.decode(self.keyencoding), Unpickler(f).load())
216n/a
217n/a
218n/aclass DbfilenameShelf(Shelf):
219n/a """Shelf implementation using the "dbm" generic dbm interface.
220n/a
221n/a This is initialized with the filename for the dbm database.
222n/a See the module's __doc__ string for an overview of the interface.
223n/a """
224n/a
225n/a def __init__(self, filename, flag='c', protocol=None, writeback=False):
226n/a import dbm
227n/a Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback)
228n/a
229n/a
230n/adef open(filename, flag='c', protocol=None, writeback=False):
231n/a """Open a persistent dictionary for reading and writing.
232n/a
233n/a The filename parameter is the base filename for the underlying
234n/a database. As a side-effect, an extension may be added to the
235n/a filename and more than one file may be created. The optional flag
236n/a parameter has the same interpretation as the flag parameter of
237n/a dbm.open(). The optional protocol parameter specifies the
238n/a version of the pickle protocol (0, 1, or 2).
239n/a
240n/a See the module's __doc__ string for an overview of the interface.
241n/a """
242n/a
243n/a return DbfilenameShelf(filename, flag, protocol, writeback)