Python code coverage for Tools/scripts/ftpmirror.py

#	count	content
1	n/a	#! /usr/bin/env python3
2	n/a
3	n/a	"""Mirror a remote ftp subtree into a local directory tree.
4	n/a
5	n/a	usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat]
6	n/a	[-l username [-p passwd [-a account]]]
7	n/a	hostname[:port] [remotedir [localdir]]
8	n/a	-v: verbose
9	n/a	-q: quiet
10	n/a	-i: interactive mode
11	n/a	-m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o')
12	n/a	-n: don't log in
13	n/a	-r: remove local files/directories no longer pertinent
14	n/a	-l username [-p passwd [-a account]]: login info (default .netrc or anonymous)
15	n/a	-s pat: skip files matching pattern
16	n/a	hostname: remote host w/ optional port separated by ':'
17	n/a	remotedir: remote directory (default initial)
18	n/a	localdir: local directory (default current)
19	n/a	"""
20	n/a
21	n/a	import os
22	n/a	import sys
23	n/a	import time
24	n/a	import getopt
25	n/a	import ftplib
26	n/a	import netrc
27	n/a	from fnmatch import fnmatch
28	n/a
29	n/a	# Print usage message and exit
30	n/a	def usage(*args):
31	n/a	sys.stdout = sys.stderr
32	n/a	for msg in args: print(msg)
33	n/a	print(__doc__)
34	n/a	sys.exit(2)
35	n/a
36	n/a	verbose = 1 # 0 for -q, 2 for -v
37	n/a	interactive = 0
38	n/a	mac = 0
39	n/a	rmok = 0
40	n/a	nologin = 0
41	n/a	skippats = ['.', '..', '.mirrorinfo']
42	n/a
43	n/a	# Main program: parse command line and start processing
44	n/a	def main():
45	n/a	global verbose, interactive, mac, rmok, nologin
46	n/a	try:
47	n/a	opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v')
48	n/a	except getopt.error as msg:
49	n/a	usage(msg)
50	n/a	login = ''
51	n/a	passwd = ''
52	n/a	account = ''
53	n/a	if not args: usage('hostname missing')
54	n/a	host = args[0]
55	n/a	port = 0
56	n/a	if ':' in host:
57	n/a	host, port = host.split(':', 1)
58	n/a	port = int(port)
59	n/a	try:
60	n/a	auth = netrc.netrc().authenticators(host)
61	n/a	if auth is not None:
62	n/a	login, account, passwd = auth
63	n/a	except (netrc.NetrcParseError, IOError):
64	n/a	pass
65	n/a	for o, a in opts:
66	n/a	if o == '-l': login = a
67	n/a	if o == '-p': passwd = a
68	n/a	if o == '-a': account = a
69	n/a	if o == '-v': verbose = verbose + 1
70	n/a	if o == '-q': verbose = 0
71	n/a	if o == '-i': interactive = 1
72	n/a	if o == '-m': mac = 1; nologin = 1; skippats.append('*.o')
73	n/a	if o == '-n': nologin = 1
74	n/a	if o == '-r': rmok = 1
75	n/a	if o == '-s': skippats.append(a)
76	n/a	remotedir = ''
77	n/a	localdir = ''
78	n/a	if args[1:]:
79	n/a	remotedir = args[1]
80	n/a	if args[2:]:
81	n/a	localdir = args[2]
82	n/a	if args[3:]: usage('too many arguments')
83	n/a	#
84	n/a	f = ftplib.FTP()
85	n/a	if verbose: print("Connecting to '%s%s'..." % (host,
86	n/a	(port and ":%d"%port or "")))
87	n/a	f.connect(host,port)
88	n/a	if not nologin:
89	n/a	if verbose:
90	n/a	print('Logging in as %r...' % (login or 'anonymous'))
91	n/a	f.login(login, passwd, account)
92	n/a	if verbose: print('OK.')
93	n/a	pwd = f.pwd()
94	n/a	if verbose > 1: print('PWD =', repr(pwd))
95	n/a	if remotedir:
96	n/a	if verbose > 1: print('cwd(%s)' % repr(remotedir))
97	n/a	f.cwd(remotedir)
98	n/a	if verbose > 1: print('OK.')
99	n/a	pwd = f.pwd()
100	n/a	if verbose > 1: print('PWD =', repr(pwd))
101	n/a	#
102	n/a	mirrorsubdir(f, localdir)
103	n/a
104	n/a	# Core logic: mirror one subdirectory (recursively)
105	n/a	def mirrorsubdir(f, localdir):
106	n/a	pwd = f.pwd()
107	n/a	if localdir and not os.path.isdir(localdir):
108	n/a	if verbose: print('Creating local directory', repr(localdir))
109	n/a	try:
110	n/a	makedir(localdir)
111	n/a	except OSError as msg:
112	n/a	print("Failed to establish local directory", repr(localdir))
113	n/a	return
114	n/a	infofilename = os.path.join(localdir, '.mirrorinfo')
115	n/a	try:
116	n/a	text = open(infofilename, 'r').read()
117	n/a	except IOError as msg:
118	n/a	text = '{}'
119	n/a	try:
120	n/a	info = eval(text)
121	n/a	except (SyntaxError, NameError):
122	n/a	print('Bad mirror info in', repr(infofilename))
123	n/a	info = {}
124	n/a	subdirs = []
125	n/a	listing = []
126	n/a	if verbose: print('Listing remote directory %r...' % (pwd,))
127	n/a	f.retrlines('LIST', listing.append)
128	n/a	filesfound = []
129	n/a	for line in listing:
130	n/a	if verbose > 1: print('-->', repr(line))
131	n/a	if mac:
132	n/a	# Mac listing has just filenames;
133	n/a	# trailing / means subdirectory
134	n/a	filename = line.strip()
135	n/a	mode = '-'
136	n/a	if filename[-1:] == '/':
137	n/a	filename = filename[:-1]
138	n/a	mode = 'd'
139	n/a	infostuff = ''
140	n/a	else:
141	n/a	# Parse, assuming a UNIX listing
142	n/a	words = line.split(None, 8)
143	n/a	if len(words) < 6:
144	n/a	if verbose > 1: print('Skipping short line')
145	n/a	continue
146	n/a	filename = words[-1].lstrip()
147	n/a	i = filename.find(" -> ")
148	n/a	if i >= 0:
149	n/a	# words[0] had better start with 'l'...
150	n/a	if verbose > 1:
151	n/a	print('Found symbolic link %r' % (filename,))
152	n/a	linkto = filename[i+4:]
153	n/a	filename = filename[:i]
154	n/a	infostuff = words[-5:-1]
155	n/a	mode = words[0]
156	n/a	skip = 0
157	n/a	for pat in skippats:
158	n/a	if fnmatch(filename, pat):
159	n/a	if verbose > 1:
160	n/a	print('Skip pattern', repr(pat), end=' ')
161	n/a	print('matches', repr(filename))
162	n/a	skip = 1
163	n/a	break
164	n/a	if skip:
165	n/a	continue
166	n/a	if mode[0] == 'd':
167	n/a	if verbose > 1:
168	n/a	print('Remembering subdirectory', repr(filename))
169	n/a	subdirs.append(filename)
170	n/a	continue
171	n/a	filesfound.append(filename)
172	n/a	if filename in info and info[filename] == infostuff:
173	n/a	if verbose > 1:
174	n/a	print('Already have this version of',repr(filename))
175	n/a	continue
176	n/a	fullname = os.path.join(localdir, filename)
177	n/a	tempname = os.path.join(localdir, '@'+filename)
178	n/a	if interactive:
179	n/a	doit = askabout('file', filename, pwd)
180	n/a	if not doit:
181	n/a	if filename not in info:
182	n/a	info[filename] = 'Not retrieved'
183	n/a	continue
184	n/a	try:
185	n/a	os.unlink(tempname)
186	n/a	except OSError:
187	n/a	pass
188	n/a	if mode[0] == 'l':
189	n/a	if verbose:
190	n/a	print("Creating symlink %r -> %r" % (filename, linkto))
191	n/a	try:
192	n/a	os.symlink(linkto, tempname)
193	n/a	except IOError as msg:
194	n/a	print("Can't create %r: %s" % (tempname, msg))
195	n/a	continue
196	n/a	else:
197	n/a	try:
198	n/a	fp = open(tempname, 'wb')
199	n/a	except IOError as msg:
200	n/a	print("Can't create %r: %s" % (tempname, msg))
201	n/a	continue
202	n/a	if verbose:
203	n/a	print('Retrieving %r from %r as %r...' % (filename, pwd, fullname))
204	n/a	if verbose:
205	n/a	fp1 = LoggingFile(fp, 1024, sys.stdout)
206	n/a	else:
207	n/a	fp1 = fp
208	n/a	t0 = time.time()
209	n/a	try:
210	n/a	f.retrbinary('RETR ' + filename,
211	n/a	fp1.write, 8*1024)
212	n/a	except ftplib.error_perm as msg:
213	n/a	print(msg)
214	n/a	t1 = time.time()
215	n/a	bytes = fp.tell()
216	n/a	fp.close()
217	n/a	if fp1 != fp:
218	n/a	fp1.close()
219	n/a	try:
220	n/a	os.unlink(fullname)
221	n/a	except OSError:
222	n/a	pass # Ignore the error
223	n/a	try:
224	n/a	os.rename(tempname, fullname)
225	n/a	except OSError as msg:
226	n/a	print("Can't rename %r to %r: %s" % (tempname, fullname, msg))
227	n/a	continue
228	n/a	info[filename] = infostuff
229	n/a	writedict(info, infofilename)
230	n/a	if verbose and mode[0] != 'l':
231	n/a	dt = t1 - t0
232	n/a	kbytes = bytes / 1024.0
233	n/a	print(int(round(kbytes)), end=' ')
234	n/a	print('Kbytes in', end=' ')
235	n/a	print(int(round(dt)), end=' ')
236	n/a	print('seconds', end=' ')
237	n/a	if t1 > t0:
238	n/a	print('(~%d Kbytes/sec)' % \
239	n/a	int(round(kbytes/dt),))
240	n/a	print()
241	n/a	#
242	n/a	# Remove files from info that are no longer remote
243	n/a	deletions = 0
244	n/a	for filename in list(info.keys()):
245	n/a	if filename not in filesfound:
246	n/a	if verbose:
247	n/a	print("Removing obsolete info entry for", end=' ')
248	n/a	print(repr(filename), "in", repr(localdir or "."))
249	n/a	del info[filename]
250	n/a	deletions = deletions + 1
251	n/a	if deletions:
252	n/a	writedict(info, infofilename)
253	n/a	#
254	n/a	# Remove local files that are no longer in the remote directory
255	n/a	try:
256	n/a	if not localdir: names = os.listdir(os.curdir)
257	n/a	else: names = os.listdir(localdir)
258	n/a	except OSError:
259	n/a	names = []
260	n/a	for name in names:
261	n/a	if name[0] == '.' or name in info or name in subdirs:
262	n/a	continue
263	n/a	skip = 0
264	n/a	for pat in skippats:
265	n/a	if fnmatch(name, pat):
266	n/a	if verbose > 1:
267	n/a	print('Skip pattern', repr(pat), end=' ')
268	n/a	print('matches', repr(name))
269	n/a	skip = 1
270	n/a	break
271	n/a	if skip:
272	n/a	continue
273	n/a	fullname = os.path.join(localdir, name)
274	n/a	if not rmok:
275	n/a	if verbose:
276	n/a	print('Local file', repr(fullname), end=' ')
277	n/a	print('is no longer pertinent')
278	n/a	continue
279	n/a	if verbose: print('Removing local file/dir', repr(fullname))
280	n/a	remove(fullname)
281	n/a	#
282	n/a	# Recursively mirror subdirectories
283	n/a	for subdir in subdirs:
284	n/a	if interactive:
285	n/a	doit = askabout('subdirectory', subdir, pwd)
286	n/a	if not doit: continue
287	n/a	if verbose: print('Processing subdirectory', repr(subdir))
288	n/a	localsubdir = os.path.join(localdir, subdir)
289	n/a	pwd = f.pwd()
290	n/a	if verbose > 1:
291	n/a	print('Remote directory now:', repr(pwd))
292	n/a	print('Remote cwd', repr(subdir))
293	n/a	try:
294	n/a	f.cwd(subdir)
295	n/a	except ftplib.error_perm as msg:
296	n/a	print("Can't chdir to", repr(subdir), ":", repr(msg))
297	n/a	else:
298	n/a	if verbose: print('Mirroring as', repr(localsubdir))
299	n/a	mirrorsubdir(f, localsubdir)
300	n/a	if verbose > 1: print('Remote cwd ..')
301	n/a	f.cwd('..')
302	n/a	newpwd = f.pwd()
303	n/a	if newpwd != pwd:
304	n/a	print('Ended up in wrong directory after cd + cd ..')
305	n/a	print('Giving up now.')
306	n/a	break
307	n/a	else:
308	n/a	if verbose > 1: print('OK.')
309	n/a
310	n/a	# Helper to remove a file or directory tree
311	n/a	def remove(fullname):
312	n/a	if os.path.isdir(fullname) and not os.path.islink(fullname):
313	n/a	try:
314	n/a	names = os.listdir(fullname)
315	n/a	except OSError:
316	n/a	names = []
317	n/a	ok = 1
318	n/a	for name in names:
319	n/a	if not remove(os.path.join(fullname, name)):
320	n/a	ok = 0
321	n/a	if not ok:
322	n/a	return 0
323	n/a	try:
324	n/a	os.rmdir(fullname)
325	n/a	except OSError as msg:
326	n/a	print("Can't remove local directory %r: %s" % (fullname, msg))
327	n/a	return 0
328	n/a	else:
329	n/a	try:
330	n/a	os.unlink(fullname)
331	n/a	except OSError as msg:
332	n/a	print("Can't remove local file %r: %s" % (fullname, msg))
333	n/a	return 0
334	n/a	return 1
335	n/a
336	n/a	# Wrapper around a file for writing to write a hash sign every block.
337	n/a	class LoggingFile:
338	n/a	def __init__(self, fp, blocksize, outfp):
339	n/a	self.fp = fp
340	n/a	self.bytes = 0
341	n/a	self.hashes = 0
342	n/a	self.blocksize = blocksize
343	n/a	self.outfp = outfp
344	n/a	def write(self, data):
345	n/a	self.bytes = self.bytes + len(data)
346	n/a	hashes = int(self.bytes) / self.blocksize
347	n/a	while hashes > self.hashes:
348	n/a	self.outfp.write('#')
349	n/a	self.outfp.flush()
350	n/a	self.hashes = self.hashes + 1
351	n/a	self.fp.write(data)
352	n/a	def close(self):
353	n/a	self.outfp.write('\n')
354	n/a
355	n/a	def raw_input(prompt):
356	n/a	sys.stdout.write(prompt)
357	n/a	sys.stdout.flush()
358	n/a	return sys.stdin.readline()
359	n/a
360	n/a	# Ask permission to download a file.
361	n/a	def askabout(filetype, filename, pwd):
362	n/a	prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd)
363	n/a	while 1:
364	n/a	reply = raw_input(prompt).strip().lower()
365	n/a	if reply in ['y', 'ye', 'yes']:
366	n/a	return 1
367	n/a	if reply in ['', 'n', 'no', 'nop', 'nope']:
368	n/a	return 0
369	n/a	print('Please answer yes or no.')
370	n/a
371	n/a	# Create a directory if it doesn't exist. Recursively create the
372	n/a	# parent directory as well if needed.
373	n/a	def makedir(pathname):
374	n/a	if os.path.isdir(pathname):
375	n/a	return
376	n/a	dirname = os.path.dirname(pathname)
377	n/a	if dirname: makedir(dirname)
378	n/a	os.mkdir(pathname, 0o777)
379	n/a
380	n/a	# Write a dictionary to a file in a way that can be read back using
381	n/a	# rval() but is still somewhat readable (i.e. not a single long line).
382	n/a	# Also creates a backup file.
383	n/a	def writedict(dict, filename):
384	n/a	dir, fname = os.path.split(filename)
385	n/a	tempname = os.path.join(dir, '@' + fname)
386	n/a	backup = os.path.join(dir, fname + '~')
387	n/a	try:
388	n/a	os.unlink(backup)
389	n/a	except OSError:
390	n/a	pass
391	n/a	fp = open(tempname, 'w')
392	n/a	fp.write('{\n')
393	n/a	for key, value in dict.items():
394	n/a	fp.write('%r: %r,\n' % (key, value))
395	n/a	fp.write('}\n')
396	n/a	fp.close()
397	n/a	try:
398	n/a	os.rename(filename, backup)
399	n/a	except OSError:
400	n/a	pass
401	n/a	os.rename(tempname, filename)
402	n/a
403	n/a
404	n/a	if __name__ == '__main__':
405	n/a	main()