»Core Development>Code coverage>Tools/iobench/iobench.py

Python code coverage for Tools/iobench/iobench.py

#countcontent
1n/a# -*- coding: utf-8 -*-
2n/a# This file should be kept compatible with both Python 2.6 and Python >= 3.0.
3n/a
4n/aimport itertools
5n/aimport os
6n/aimport platform
7n/aimport re
8n/aimport sys
9n/aimport time
10n/afrom optparse import OptionParser
11n/a
12n/aout = sys.stdout
13n/a
14n/aTEXT_ENCODING = 'utf8'
15n/aNEWLINES = 'lf'
16n/a
17n/a# Compatibility
18n/atry:
19n/a xrange
20n/aexcept NameError:
21n/a xrange = range
22n/a
23n/adef text_open(fn, mode, encoding=None):
24n/a try:
25n/a return open(fn, mode, encoding=encoding or TEXT_ENCODING)
26n/a except TypeError:
27n/a if 'r' in mode:
28n/a mode += 'U' # 'U' mode is needed only in Python 2.x
29n/a return open(fn, mode)
30n/a
31n/adef get_file_sizes():
32n/a for s in ['20 KB', '400 KB', '10 MB']:
33n/a size, unit = s.split()
34n/a size = int(size) * {'KB': 1024, 'MB': 1024 ** 2}[unit]
35n/a yield s.replace(' ', ''), size
36n/a
37n/adef get_binary_files():
38n/a return ((name + ".bin", size) for name, size in get_file_sizes())
39n/a
40n/adef get_text_files():
41n/a return (("%s-%s-%s.txt" % (name, TEXT_ENCODING, NEWLINES), size)
42n/a for name, size in get_file_sizes())
43n/a
44n/adef with_open_mode(mode):
45n/a def decorate(f):
46n/a f.file_open_mode = mode
47n/a return f
48n/a return decorate
49n/a
50n/adef with_sizes(*sizes):
51n/a def decorate(f):
52n/a f.file_sizes = sizes
53n/a return f
54n/a return decorate
55n/a
56n/a
57n/a# Here begin the tests
58n/a
59n/a@with_open_mode("r")
60n/a@with_sizes("medium")
61n/adef read_bytewise(f):
62n/a """ read one unit at a time """
63n/a f.seek(0)
64n/a while f.read(1):
65n/a pass
66n/a
67n/a@with_open_mode("r")
68n/a@with_sizes("medium")
69n/adef read_small_chunks(f):
70n/a """ read 20 units at a time """
71n/a f.seek(0)
72n/a while f.read(20):
73n/a pass
74n/a
75n/a@with_open_mode("r")
76n/a@with_sizes("medium")
77n/adef read_big_chunks(f):
78n/a """ read 4096 units at a time """
79n/a f.seek(0)
80n/a while f.read(4096):
81n/a pass
82n/a
83n/a@with_open_mode("r")
84n/a@with_sizes("small", "medium", "large")
85n/adef read_whole_file(f):
86n/a """ read whole contents at once """
87n/a f.seek(0)
88n/a while f.read():
89n/a pass
90n/a
91n/a@with_open_mode("rt")
92n/a@with_sizes("medium")
93n/adef read_lines(f):
94n/a """ read one line at a time """
95n/a f.seek(0)
96n/a for line in f:
97n/a pass
98n/a
99n/a@with_open_mode("r")
100n/a@with_sizes("medium")
101n/adef seek_forward_bytewise(f):
102n/a """ seek forward one unit at a time """
103n/a f.seek(0, 2)
104n/a size = f.tell()
105n/a f.seek(0, 0)
106n/a for i in xrange(0, size - 1):
107n/a f.seek(i, 0)
108n/a
109n/a@with_open_mode("r")
110n/a@with_sizes("medium")
111n/adef seek_forward_blockwise(f):
112n/a """ seek forward 1000 units at a time """
113n/a f.seek(0, 2)
114n/a size = f.tell()
115n/a f.seek(0, 0)
116n/a for i in xrange(0, size - 1, 1000):
117n/a f.seek(i, 0)
118n/a
119n/a@with_open_mode("rb")
120n/a@with_sizes("medium")
121n/adef read_seek_bytewise(f):
122n/a """ alternate read & seek one unit """
123n/a f.seek(0)
124n/a while f.read(1):
125n/a f.seek(1, 1)
126n/a
127n/a@with_open_mode("rb")
128n/a@with_sizes("medium")
129n/adef read_seek_blockwise(f):
130n/a """ alternate read & seek 1000 units """
131n/a f.seek(0)
132n/a while f.read(1000):
133n/a f.seek(1000, 1)
134n/a
135n/a
136n/a@with_open_mode("w")
137n/a@with_sizes("small")
138n/adef write_bytewise(f, source):
139n/a """ write one unit at a time """
140n/a for i in xrange(0, len(source)):
141n/a f.write(source[i:i+1])
142n/a
143n/a@with_open_mode("w")
144n/a@with_sizes("medium")
145n/adef write_small_chunks(f, source):
146n/a """ write 20 units at a time """
147n/a for i in xrange(0, len(source), 20):
148n/a f.write(source[i:i+20])
149n/a
150n/a@with_open_mode("w")
151n/a@with_sizes("medium")
152n/adef write_medium_chunks(f, source):
153n/a """ write 4096 units at a time """
154n/a for i in xrange(0, len(source), 4096):
155n/a f.write(source[i:i+4096])
156n/a
157n/a@with_open_mode("w")
158n/a@with_sizes("large")
159n/adef write_large_chunks(f, source):
160n/a """ write 1e6 units at a time """
161n/a for i in xrange(0, len(source), 1000000):
162n/a f.write(source[i:i+1000000])
163n/a
164n/a
165n/a@with_open_mode("w+")
166n/a@with_sizes("small")
167n/adef modify_bytewise(f, source):
168n/a """ modify one unit at a time """
169n/a f.seek(0)
170n/a for i in xrange(0, len(source)):
171n/a f.write(source[i:i+1])
172n/a
173n/a@with_open_mode("w+")
174n/a@with_sizes("medium")
175n/adef modify_small_chunks(f, source):
176n/a """ modify 20 units at a time """
177n/a f.seek(0)
178n/a for i in xrange(0, len(source), 20):
179n/a f.write(source[i:i+20])
180n/a
181n/a@with_open_mode("w+")
182n/a@with_sizes("medium")
183n/adef modify_medium_chunks(f, source):
184n/a """ modify 4096 units at a time """
185n/a f.seek(0)
186n/a for i in xrange(0, len(source), 4096):
187n/a f.write(source[i:i+4096])
188n/a
189n/a@with_open_mode("wb+")
190n/a@with_sizes("medium")
191n/adef modify_seek_forward_bytewise(f, source):
192n/a """ alternate write & seek one unit """
193n/a f.seek(0)
194n/a for i in xrange(0, len(source), 2):
195n/a f.write(source[i:i+1])
196n/a f.seek(i+2)
197n/a
198n/a@with_open_mode("wb+")
199n/a@with_sizes("medium")
200n/adef modify_seek_forward_blockwise(f, source):
201n/a """ alternate write & seek 1000 units """
202n/a f.seek(0)
203n/a for i in xrange(0, len(source), 2000):
204n/a f.write(source[i:i+1000])
205n/a f.seek(i+2000)
206n/a
207n/a# XXX the 2 following tests don't work with py3k's text IO
208n/a@with_open_mode("wb+")
209n/a@with_sizes("medium")
210n/adef read_modify_bytewise(f, source):
211n/a """ alternate read & write one unit """
212n/a f.seek(0)
213n/a for i in xrange(0, len(source), 2):
214n/a f.read(1)
215n/a f.write(source[i+1:i+2])
216n/a
217n/a@with_open_mode("wb+")
218n/a@with_sizes("medium")
219n/adef read_modify_blockwise(f, source):
220n/a """ alternate read & write 1000 units """
221n/a f.seek(0)
222n/a for i in xrange(0, len(source), 2000):
223n/a f.read(1000)
224n/a f.write(source[i+1000:i+2000])
225n/a
226n/a
227n/aread_tests = [
228n/a read_bytewise, read_small_chunks, read_lines, read_big_chunks,
229n/a None, read_whole_file, None,
230n/a seek_forward_bytewise, seek_forward_blockwise,
231n/a read_seek_bytewise, read_seek_blockwise,
232n/a]
233n/a
234n/awrite_tests = [
235n/a write_bytewise, write_small_chunks, write_medium_chunks, write_large_chunks,
236n/a]
237n/a
238n/amodify_tests = [
239n/a modify_bytewise, modify_small_chunks, modify_medium_chunks,
240n/a None,
241n/a modify_seek_forward_bytewise, modify_seek_forward_blockwise,
242n/a read_modify_bytewise, read_modify_blockwise,
243n/a]
244n/a
245n/adef run_during(duration, func):
246n/a _t = time.time
247n/a n = 0
248n/a start = os.times()
249n/a start_timestamp = _t()
250n/a real_start = start[4] or start_timestamp
251n/a while True:
252n/a func()
253n/a n += 1
254n/a if _t() - start_timestamp > duration:
255n/a break
256n/a end = os.times()
257n/a real = (end[4] if start[4] else time.time()) - real_start
258n/a return n, real, sum(end[0:2]) - sum(start[0:2])
259n/a
260n/adef warm_cache(filename):
261n/a with open(filename, "rb") as f:
262n/a f.read()
263n/a
264n/a
265n/adef run_all_tests(options):
266n/a def print_label(filename, func):
267n/a name = re.split(r'[-.]', filename)[0]
268n/a out.write(
269n/a ("[%s] %s... "
270n/a % (name.center(7), func.__doc__.strip())
271n/a ).ljust(52))
272n/a out.flush()
273n/a
274n/a def print_results(size, n, real, cpu):
275n/a bw = n * float(size) / 1024 ** 2 / real
276n/a bw = ("%4d MB/s" if bw > 100 else "%.3g MB/s") % bw
277n/a out.write(bw.rjust(12) + "\n")
278n/a if cpu < 0.90 * real:
279n/a out.write(" warning: test above used only %d%% CPU, "
280n/a "result may be flawed!\n" % (100.0 * cpu / real))
281n/a
282n/a def run_one_test(name, size, open_func, test_func, *args):
283n/a mode = test_func.file_open_mode
284n/a print_label(name, test_func)
285n/a if "w" not in mode or "+" in mode:
286n/a warm_cache(name)
287n/a with open_func(name) as f:
288n/a n, real, cpu = run_during(1.5, lambda: test_func(f, *args))
289n/a print_results(size, n, real, cpu)
290n/a
291n/a def run_test_family(tests, mode_filter, files, open_func, *make_args):
292n/a for test_func in tests:
293n/a if test_func is None:
294n/a out.write("\n")
295n/a continue
296n/a if mode_filter in test_func.file_open_mode:
297n/a continue
298n/a for s in test_func.file_sizes:
299n/a name, size = files[size_names[s]]
300n/a #name += file_ext
301n/a args = tuple(f(name, size) for f in make_args)
302n/a run_one_test(name, size,
303n/a open_func, test_func, *args)
304n/a
305n/a size_names = {
306n/a "small": 0,
307n/a "medium": 1,
308n/a "large": 2,
309n/a }
310n/a
311n/a print("Python %s" % sys.version)
312n/a if sys.version_info < (3, 3):
313n/a if sys.maxunicode > 0xffff:
314n/a text = "UCS-4 (wide build)"
315n/a else:
316n/a text = "UTF-16 (narrow build)"
317n/a else:
318n/a text = "PEP 393"
319n/a print("Unicode: %s" % text)
320n/a print(platform.platform())
321n/a binary_files = list(get_binary_files())
322n/a text_files = list(get_text_files())
323n/a if "b" in options:
324n/a print("Binary unit = one byte")
325n/a if "t" in options:
326n/a print("Text unit = one character (%s-decoded)" % TEXT_ENCODING)
327n/a
328n/a # Binary reads
329n/a if "b" in options and "r" in options:
330n/a print("\n** Binary input **\n")
331n/a run_test_family(read_tests, "t", binary_files, lambda fn: open(fn, "rb"))
332n/a
333n/a # Text reads
334n/a if "t" in options and "r" in options:
335n/a print("\n** Text input **\n")
336n/a run_test_family(read_tests, "b", text_files, lambda fn: text_open(fn, "r"))
337n/a
338n/a # Binary writes
339n/a if "b" in options and "w" in options:
340n/a print("\n** Binary append **\n")
341n/a def make_test_source(name, size):
342n/a with open(name, "rb") as f:
343n/a return f.read()
344n/a run_test_family(write_tests, "t", binary_files,
345n/a lambda fn: open(os.devnull, "wb"), make_test_source)
346n/a
347n/a # Text writes
348n/a if "t" in options and "w" in options:
349n/a print("\n** Text append **\n")
350n/a def make_test_source(name, size):
351n/a with text_open(name, "r") as f:
352n/a return f.read()
353n/a run_test_family(write_tests, "b", text_files,
354n/a lambda fn: text_open(os.devnull, "w"), make_test_source)
355n/a
356n/a # Binary overwrites
357n/a if "b" in options and "w" in options:
358n/a print("\n** Binary overwrite **\n")
359n/a def make_test_source(name, size):
360n/a with open(name, "rb") as f:
361n/a return f.read()
362n/a run_test_family(modify_tests, "t", binary_files,
363n/a lambda fn: open(fn, "r+b"), make_test_source)
364n/a
365n/a # Text overwrites
366n/a if "t" in options and "w" in options:
367n/a print("\n** Text overwrite **\n")
368n/a def make_test_source(name, size):
369n/a with text_open(name, "r") as f:
370n/a return f.read()
371n/a run_test_family(modify_tests, "b", text_files,
372n/a lambda fn: text_open(fn, "r+"), make_test_source)
373n/a
374n/a
375n/adef prepare_files():
376n/a print("Preparing files...")
377n/a # Binary files
378n/a for name, size in get_binary_files():
379n/a if os.path.isfile(name) and os.path.getsize(name) == size:
380n/a continue
381n/a with open(name, "wb") as f:
382n/a f.write(os.urandom(size))
383n/a # Text files
384n/a chunk = []
385n/a with text_open(__file__, "r", encoding='utf8') as f:
386n/a for line in f:
387n/a if line.startswith("# <iobench text chunk marker>"):
388n/a break
389n/a else:
390n/a raise RuntimeError(
391n/a "Couldn't find chunk marker in %s !" % __file__)
392n/a if NEWLINES == "all":
393n/a it = itertools.cycle(["\n", "\r", "\r\n"])
394n/a else:
395n/a it = itertools.repeat(
396n/a {"cr": "\r", "lf": "\n", "crlf": "\r\n"}[NEWLINES])
397n/a chunk = "".join(line.replace("\n", next(it)) for line in f)
398n/a if isinstance(chunk, bytes):
399n/a chunk = chunk.decode('utf8')
400n/a chunk = chunk.encode(TEXT_ENCODING)
401n/a for name, size in get_text_files():
402n/a if os.path.isfile(name) and os.path.getsize(name) == size:
403n/a continue
404n/a head = chunk * (size // len(chunk))
405n/a tail = chunk[:size % len(chunk)]
406n/a # Adjust tail to end on a character boundary
407n/a while True:
408n/a try:
409n/a tail.decode(TEXT_ENCODING)
410n/a break
411n/a except UnicodeDecodeError:
412n/a tail = tail[:-1]
413n/a with open(name, "wb") as f:
414n/a f.write(head)
415n/a f.write(tail)
416n/a
417n/adef main():
418n/a global TEXT_ENCODING, NEWLINES
419n/a
420n/a usage = "usage: %prog [-h|--help] [options]"
421n/a parser = OptionParser(usage=usage)
422n/a parser.add_option("-b", "--binary",
423n/a action="store_true", dest="binary", default=False,
424n/a help="run binary I/O tests")
425n/a parser.add_option("-t", "--text",
426n/a action="store_true", dest="text", default=False,
427n/a help="run text I/O tests")
428n/a parser.add_option("-r", "--read",
429n/a action="store_true", dest="read", default=False,
430n/a help="run read tests")
431n/a parser.add_option("-w", "--write",
432n/a action="store_true", dest="write", default=False,
433n/a help="run write & modify tests")
434n/a parser.add_option("-E", "--encoding",
435n/a action="store", dest="encoding", default=None,
436n/a help="encoding for text tests (default: %s)" % TEXT_ENCODING)
437n/a parser.add_option("-N", "--newlines",
438n/a action="store", dest="newlines", default='lf',
439n/a help="line endings for text tests "
440n/a "(one of: {lf (default), cr, crlf, all})")
441n/a parser.add_option("-m", "--io-module",
442n/a action="store", dest="io_module", default=None,
443n/a help="io module to test (default: builtin open())")
444n/a options, args = parser.parse_args()
445n/a if args:
446n/a parser.error("unexpected arguments")
447n/a NEWLINES = options.newlines.lower()
448n/a if NEWLINES not in ('lf', 'cr', 'crlf', 'all'):
449n/a parser.error("invalid 'newlines' option: %r" % NEWLINES)
450n/a
451n/a test_options = ""
452n/a if options.read:
453n/a test_options += "r"
454n/a if options.write:
455n/a test_options += "w"
456n/a elif not options.read:
457n/a test_options += "rw"
458n/a if options.text:
459n/a test_options += "t"
460n/a if options.binary:
461n/a test_options += "b"
462n/a elif not options.text:
463n/a test_options += "tb"
464n/a
465n/a if options.encoding:
466n/a TEXT_ENCODING = options.encoding
467n/a
468n/a if options.io_module:
469n/a globals()['open'] = __import__(options.io_module, {}, {}, ['open']).open
470n/a
471n/a prepare_files()
472n/a run_all_tests(test_options)
473n/a
474n/aif __name__ == "__main__":
475n/a main()
476n/a
477n/a
478n/a# -- This part to exercise text reading. Don't change anything! --
479n/a# <iobench text chunk marker>
480n/a
481n/a"""
482n/a1.
483n/aGáttir allar,
484n/aáðr gangi fram,
485n/aum skoðask skyli,
486n/aum skyggnast skyli,
487n/aþví at óvíst er at vita,
488n/ahvar óvinir
489n/asitja á fleti fyrir.
490n/a
491n/a2.
492n/aGefendr heilir!
493n/aGestr er inn kominn,
494n/ahvar skal sitja sjá?
495n/aMjök er bráðr,
496n/asá er á bröndum skal
497n/asíns of freista frama.
498n/a
499n/a3.
500n/aElds er þörf,
501n/aþeims inn er kominn
502n/aok á kné kalinn;
503n/amatar ok váða
504n/aer manni þörf,
505n/aþeim er hefr um fjall farit.
506n/a
507n/a4.
508n/aVatns er þörf,
509n/aþeim er til verðar kemr,
510n/aþerru ok þjóðlaðar,
511n/agóðs of æðis,
512n/aef sér geta mætti,
513n/aorðs ok endrþögu.
514n/a
515n/a5.
516n/aVits er þörf,
517n/aþeim er víða ratar;
518n/adælt er heima hvat;
519n/aat augabragði verðr,
520n/asá er ekki kann
521n/aok með snotrum sitr.
522n/a
523n/a6.
524n/aAt hyggjandi sinni
525n/askyli-t maðr hræsinn vera,
526n/aheldr gætinn at geði;
527n/aþá er horskr ok þögull
528n/akemr heimisgarða til,
529n/asjaldan verðr víti vörum,
530n/aþví at óbrigðra vin
531n/afær maðr aldregi
532n/aen mannvit mikit.
533n/a
534n/a7.
535n/aInn vari gestr,
536n/aer til verðar kemr,
537n/aþunnu hljóði þegir,
538n/aeyrum hlýðir,
539n/aen augum skoðar;
540n/asvá nýsisk fróðra hverr fyrir.
541n/a
542n/a8.
543n/aHinn er sæll,
544n/aer sér of getr
545n/alof ok líknstafi;
546n/aódælla er við þat,
547n/aer maðr eiga skal
548n/aannars brjóstum í.
549n/a"""
550n/a
551n/a"""
552n/aC'est revenir tard, je le sens, sur un sujet trop rebattu et déjà presque oublié. Mon état, qui ne me permet plus aucun travail suivi, mon aversion pour le genre polémique, ont causé ma lenteur à écrire et ma répugnance à publier. J'aurais même tout à fait supprimé ces Lettres, ou plutôt je lie les aurais point écrites, s'il n'eût été question que de moi : Mais ma patrie ne m'est pas tellement devenue étrangère que je puisse voir tranquillement opprimer ses citoyens, surtout lorsqu'ils n'ont compromis leurs droits qu'en défendant ma cause. Je serais le dernier des hommes si dans une telle occasion j'écoutais un sentiment qui n'est plus ni douceur ni patience, mais faiblesse et lâcheté, dans celui qu'il empêche de remplir son devoir.
553n/aRien de moins important pour le public, j'en conviens, que la matière de ces lettres. La constitution d'une petite République, le sort d'un petit particulier, l'exposé de quelques injustices, la réfutation de quelques sophismes ; tout cela n'a rien en soi d'assez considérable pour mériter beaucoup de lecteurs : mais si mes sujets sont petits mes objets sont grands, et dignes de l'attention de tout honnête homme. Laissons Genève à sa place, et Rousseau dans sa dépression ; mais la religion, mais la liberté, la justice ! voilà, qui que vous soyez, ce qui n'est pas au-dessous de vous.
554n/aQu'on ne cherche pas même ici dans le style le dédommagement de l'aridité de la matière. Ceux que quelques traits heureux de ma plume ont si fort irrités trouveront de quoi s'apaiser dans ces lettres, L'honneur de défendre un opprimé eût enflammé mon coeur si j'avais parlé pour un autre. Réduit au triste emploi de me défendre moi-même, j'ai dû me borner à raisonner ; m'échauffer eût été m'avilir. J'aurai donc trouvé grâce en ce point devant ceux qui s'imaginent qu'il est essentiel à la vérité d'être dite froidement ; opinion que pourtant j'ai peine à comprendre. Lorsqu'une vive persuasion nous anime, le moyen d'employer un langage glacé ? Quand Archimède tout transporté courait nu dans les rues de Syracuse, en avait-il moins trouvé la vérité parce qu'il se passionnait pour elle ? Tout au contraire, celui qui la sent ne peut s'abstenir de l'adorer ; celui qui demeure froid ne l'a pas vue.
555n/aQuoi qu'il en soit, je prie les lecteurs de vouloir bien mettre à part mon beau style, et d'examiner seulement si je raisonne bien ou mal ; car enfin, de cela seul qu'un auteur s'exprime en bons termes, je ne vois pas comment il peut s'ensuivre que cet auteur ne sait ce qu'il dit.
556n/a"""