2 """Recursive FTP scanners"""
5 from __future__ import print_function
8 from .ftpparse import ftpparse
11 class FtpScanError(Exception): pass
12 ftpscan_error_mark = object() # error marker
19 def __call__(self, line):
20 entry = ftpparse(line)
22 self.entries.append(entry)
24 def filter(self, file_type):
25 return filter(lambda e, file_type=file_type: e.file_type == file_type,
29 return self.filter('f')
31 def directories(self):
32 return filter(lambda e: e.name not in (".", ".."), self.filter('d'))
35 class ReconnectingFTPCallWrapper:
36 retries = 10 # retries per function call
38 def __init__(self, wrapper, func):
39 self.wrapper = wrapper
42 def __call__(self, *params, **kw):
43 wrapper = self.wrapper
46 for retry in range(self.retries):
48 return func(*params, **kw)
52 ftp_dir = wrapper._ftp_dir
53 wrapper._tree.append((ftpscan_error_mark, "Connection reset by peer at directory `%s'. Reconnecting..." % ftp_dir))
58 ftp.connect(wrapper._ftp_server, wrapper._ftp_port)
59 ftp.login(wrapper._login, wrapper._password)
62 class ReconnectingFTPWrapper:
63 ReconnectingFTPCallWrapperClass = ReconnectingFTPCallWrapper
65 def __init__(self, ftp, ftp_server, ftp_port=0, login=None, password=None, ftp_dir='/', tree=None):
67 self._ftp_server = ftp_server
68 self._ftp_port = ftp_port
70 self._password = password
71 ftp_dir = [''] + [name for name in ftp_dir.split('/') if name] # remove double slashes //
72 self._ftp_dir = '/'.join(ftp_dir)
75 def cwd(self, new_cwd, do_ftp=True):
76 ftp_dir = self._ftp_dir.split('/')
80 ftp_dir.append(new_cwd)
81 self._ftp_dir = '/'.join(ftp_dir)
82 if do_ftp: self._wrap(self._ftp.cwd)(new_cwd)
84 def __getattr__(self, attr):
85 value = getattr(self._ftp, attr)
87 return self._wrap(value)
90 def _wrap(self, func):
91 return self.ReconnectingFTPCallWrapperClass(self, func)
94 def _traverse_ftp(ftp, tree, ftp_dir):
95 get_files = GetFiles()
98 except ftplib.all_errors as msg:
99 tree.append((ftpscan_error_mark, "Cannot list directory `%s': %s" % (ftp_dir, msg)))
101 files = get_files.files()
102 directories = get_files.directories()
104 if ftp_dir and ftp_dir[-1] == '/':
105 ftp_dir = ftp_dir[:-1] # Prevent paths to contain double slashes //
107 tree.append((ftp_dir, files))
109 for d in directories:
111 full_path = ftp_dir + '/' + name
114 except ftplib.error_perm as msg:
115 tree.append((ftpscan_error_mark, "Cannot enter directory `%s': %s" % (full_path, msg)))
116 if isinstance(ftp, ReconnectingFTPWrapper):
118 except ftplib.all_errors as msg:
119 tree.append((ftpscan_error_mark, "Cannot enter directory `%s': %s" % (full_path, msg)))
121 _traverse_ftp(ftp, tree, full_path)
125 def ftpscan1(ftp_server, ftp_port=0, login=None, password=None,
126 ftp_dir='/', passive=None, FTPClass=ftplib.FTP, reconnect=False,
127 ReconnectingFTPWrapperClass=ReconnectingFTPWrapper):
128 """Recursive FTP scan using one-by-one directory traversing. It is slow
129 but robust - it works with all but very broken FTP servers.
133 if passive is not None:
134 ftp.set_pasv(passive)
136 ftp = ReconnectingFTPWrapperClass(ftp, ftp_server, ftp_port, login, password, ftp_dir, tree)
137 ftp.connect(ftp_server, ftp_port)
138 ftp.login(login, password)
142 _traverse_ftp(ftp, tree, ftp_dir)
148 def ftpscanrecursive(ftp_server, ftp_port=0, login=None, password=None,
149 ftp_dir='/', passive=None, FTPClass=ftplib.FTP, reconnect=False):
151 Recursive FTP scan using fast LIST -R command. Not all servers supports
155 if passive is not None:
156 ftp.set_pasv(passive)
157 ftp.connect(ftp_server, ftp_port)
158 ftp.login(login, password)
164 ftp.dir("-R", lines.append)
165 except ftplib.error_perm:
166 # The server does not implement LIST -R and
167 # treats -R as a name of a directory (-:
169 raise FtpScanError("the server does not implement recursive listing")
173 current_dir = ftp_dir
178 if line[-1] == ':' and not line.startswith("-rw-"): # directory
179 tree.append((current_dir, files))
181 line = line[1:] # remove leading dot
184 current_dir = line[:-1]
187 if not line.startswith("total "):
188 entry = ftpparse(line)
190 if entry.file_type == 'f':
193 tree.append((ftpscan_error_mark, "Unrecognised line: `%s'" % line))
194 tree.append((current_dir, files))
197 raise FtpScanError("the server ignores -R in LIST")
202 def ftpscan(ftp_server, ftp_port=0, login=None, password=None,
203 ftp_dir='/', passive=None, FTPClass=ftplib.FTP):
205 return ftpscanrecursive(ftp_server, ftp_port, login, password, ftp_dir, passive, FTPClass)
208 return ftpscan1(ftp_server, ftp_port, login, password, ftp_dir, passive, FTPClass)
210 return ftpscan1(ftp_server, ftp_port, login, password, ftp_dir, passive, FTPClass, True)
212 return ftpscan1(ftp_server, ftp_port, login, password, ftp_dir, passive, FTPClass, True)
215 def test(ftp_server, func, passive=None, reconnect=False):
216 from time import time
219 tree = func(ftp_server, passive=passive, reconnect=reconnect)
222 print(stop_time - start_time)
224 logfname = "%s.list" % ftp_server
225 log = codecs.open(logfname, 'w', encoding='utf-8')
227 for ftp_dir, files in tree:
228 if ftp_dir == ftpscan_error_mark:
229 log.write("Error:\n")
233 log.write(ftp_dir + '\n')
236 log.write(_file.name + '\n')
240 sys.stderr.write("Usage: %s [-a|-p] [hostname]\n" % sys.argv[0])
243 if __name__ == "__main__":
245 from getopt import getopt, GetoptError
248 options, arguments = getopt(sys.argv[1:], "hap",
249 ["help", "active", "passive"])
255 for option, value in options:
256 if option in ("-h", "--help"):
258 elif option in ("-a", "--active"):
260 elif option in ("-p", "--passive"):
267 ftp_server = "localhost"
271 ftp_server = arguments[0]
273 print("Scanning", ftp_server)
275 test(ftp_server, ftpscanrecursive, passive)
276 except FtpScanError as msg:
277 print("Rescanning due to the error:", msg)
279 test(ftp_server, ftpscan1, passive)
281 print("Rescanning due to the error: connection reset by peer")
282 test(ftp_server, ftpscan1, passive, True)
284 print("Rescanning due to the error: connection reset by peer")
285 test(ftp_server, ftpscan1, passive, True)