2 """Recursive FTP scanners"""
5 from __future__ import print_function
7 from m_lib.net.ftp.ftpparse import ftpparse
10 class FtpScanError(Exception): pass
11 ftpscan_error_mark = object() # error marker
18 def __call__(self, line):
19 entry = ftpparse(line)
21 self.entries.append(entry)
23 def filter(self, file_type):
24 return filter(lambda e, file_type=file_type: e.file_type == file_type,
28 return self.filter('f')
30 def directories(self):
31 return filter(lambda e: e.name not in (".", ".."), self.filter('d'))
34 class ReconnectingFTPCallWrapper:
35 retries = 10 # retries per function call
37 def __init__(self, wrapper, func):
38 self.wrapper = wrapper
41 def __call__(self, *params, **kw):
42 wrapper = self.wrapper
45 for retry in range(self.retries):
47 return func(*params, **kw)
51 ftp_dir = wrapper._ftp_dir
52 wrapper._tree.append((ftpscan_error_mark, "Connection reset by peer at directory `%s'. Reconnecting..." % ftp_dir))
57 ftp.connect(wrapper._ftp_server, wrapper._ftp_port)
58 ftp.login(wrapper._login, wrapper._password)
61 class ReconnectingFTPWrapper:
62 ReconnectingFTPCallWrapperClass = ReconnectingFTPCallWrapper
64 def __init__(self, ftp, ftp_server, ftp_port=None, login=None, password=None, ftp_dir='/', tree=None):
66 self._ftp_server = ftp_server
67 self._ftp_port = ftp_port
69 self._password = password
70 ftp_dir = [''] + [name for name in ftp_dir.split('/') if name] # remove double slashes //
71 self._ftp_dir = '/'.join(ftp_dir)
74 def cwd(self, new_cwd, do_ftp=True):
75 ftp_dir = self._ftp_dir.split('/')
79 ftp_dir.append(new_cwd)
80 self._ftp_dir = '/'.join(ftp_dir)
81 if do_ftp: self._wrap(self._ftp.cwd)(new_cwd)
83 def __getattr__(self, attr):
84 value = getattr(self._ftp, attr)
86 return self._wrap(value)
89 def _wrap(self, func):
90 return self.ReconnectingFTPCallWrapperClass(self, func)
93 def _traverse_ftp(ftp, tree, ftp_dir):
94 get_files = GetFiles()
97 except ftplib.all_errors as msg:
98 tree.append((ftpscan_error_mark, "Cannot list directory `%s': %s" % (ftp_dir, msg)))
100 files = get_files.files()
101 directories = get_files.directories()
103 if ftp_dir and ftp_dir[-1] == '/':
104 ftp_dir = ftp_dir[:-1] # Prevent paths to contain double slashes //
106 tree.append((ftp_dir, files))
108 for d in directories:
110 full_path = ftp_dir + '/' + name
113 except ftplib.error_perm as msg:
114 tree.append((ftpscan_error_mark, "Cannot enter directory `%s': %s" % (full_path, msg)))
115 if isinstance(ftp, ReconnectingFTPWrapper):
117 except ftplib.all_errors as msg:
118 tree.append((ftpscan_error_mark, "Cannot enter directory `%s': %s" % (full_path, msg)))
120 _traverse_ftp(ftp, tree, full_path)
124 def ftpscan1(ftp_server, ftp_port=None, login=None, password=None,
125 ftp_dir='/', passive=None, FTPClass=ftplib.FTP, reconnect=False,
126 ReconnectingFTPWrapperClass=ReconnectingFTPWrapper):
127 """Recursive FTP scan using one-by-one directory traversing. It is slow
128 but robust - it works with all but very broken FTP servers.
132 if passive is not None:
133 ftp.set_pasv(passive)
135 ftp = ReconnectingFTPWrapperClass(ftp, ftp_server, ftp_port, login, password, ftp_dir, tree)
136 ftp.connect(ftp_server, ftp_port)
137 ftp.login(login, password)
141 _traverse_ftp(ftp, tree, ftp_dir)
147 def ftpscanrecursive(ftp_server, ftp_port=None, login=None, password=None,
148 ftp_dir='/', passive=None, FTPClass=ftplib.FTP, reconnect=False):
150 Recursive FTP scan using fast LIST -R command. Not all servers supports
154 if passive is not None:
155 ftp.set_pasv(passive)
156 ftp.connect(ftp_server, ftp_port)
157 ftp.login(login, password)
163 ftp.dir("-R", lines.append)
164 except ftplib.error_perm:
165 # The server does not implement LIST -R and
166 # treats -R as a name of a directory (-:
168 raise FtpScanError, "the server does not implement recursive listing"
172 current_dir = ftp_dir
177 if line[-1] == ':' and not line.startswith("-rw-"): # directory
178 tree.append((current_dir, files))
180 line = line[1:] # remove leading dot
183 current_dir = line[:-1]
186 if not line.startswith("total "):
187 entry = ftpparse(line)
189 if entry.file_type == 'f':
192 tree.append((ftpscan_error_mark, "Unrecognised line: `%s'" % line))
193 tree.append((current_dir, files))
196 raise FtpScanError, "the server ignores -R in LIST"
201 def ftpscan(ftp_server, ftp_port=None, login=None, password=None,
202 ftp_dir='/', passive=None, FTPClass=ftplib.FTP):
204 return ftpscanrecursive(ftp_server, ftp_port, login, password, ftp_dir, passive, FTPClass)
207 return ftpscan1(ftp_server, ftp_port, login, password, ftp_dir, passive, FTPClass)
209 return ftpscan1(ftp_server, ftp_port, login, password, ftp_dir, passive, FTPClass, True)
211 return ftpscan1(ftp_server, ftp_port, login, password, ftp_dir, passive, FTPClass, True)
214 def test(ftp_server, func, passive=None, reconnect=False):
215 from time import time
218 tree = func(ftp_server, passive=passive, reconnect=reconnect)
221 print(stop_time - start_time)
223 logfname = "%s.list" % ftp_server
224 log = open(logfname, 'w')
226 for ftp_dir, files in tree:
227 if ftp_dir == ftpscan_error_mark:
228 log.write("Error:\n")
232 log.write(ftp_dir + '\n')
235 log.write(_file.name + '\n')
239 sys.stderr.write("Usage: %s [-a|-p] [hostname]\n" % sys.argv[0])
242 if __name__ == "__main__":
244 from getopt import getopt, GetoptError
247 options, arguments = getopt(sys.argv[1:], "hap",
248 ["help", "active", "passive"])
254 for option, value in options:
255 if option in ("-h", "--help"):
257 elif option in ("-a", "--active"):
259 elif option in ("-p", "--passive"):
266 ftp_server = "localhost"
270 ftp_server = arguments[0]
272 print("Scanning", ftp_server)
274 test(ftp_server, ftpscanrecursive, passive)
275 except FtpScanError as msg:
276 print("Rescanning due to the error:", msg)
278 test(ftp_server, ftpscan1, passive)
280 print("Rescanning due to the error: connection reset by peer")
281 test(ftp_server, ftpscan1, passive, True)
283 print("Rescanning due to the error: connection reset by peer")
284 test(ftp_server, ftpscan1, passive, True)