X-Git-Url: https://git.phdru.name/?a=blobdiff_plain;f=Robots%2Fbkmk_rurllib.py;h=55e0037e8999e261010d095af4eb1a9b61ad9852;hb=22d567ee3a7eac29b4c83eb1344f7bae4ef0bdb1;hp=018896379ab4a4f2ca2bd59f10736bb975314777;hpb=d5365dbb38d672f256a5e7a2f949c789bc5793ce;p=bookmarks_db.git diff --git a/Robots/bkmk_rurllib.py b/Robots/bkmk_rurllib.py index 0188963..55e0037 100644 --- a/Robots/bkmk_rurllib.py +++ b/Robots/bkmk_rurllib.py @@ -11,8 +11,9 @@ __license__ = "GNU GPL" __all__ = ['robot_urllib'] -import sys, os -import time, urllib +import sys +import urllib + from Robots.bkmk_robot_base import robot_base, get_error @@ -22,6 +23,7 @@ class RedirectException(Exception): self.errcode = errcode self.newurl = newurl + class MyURLopener(urllib.URLopener): # Error 301 -- relocated (permanently) def http_error_301(self, url, fp, errcode, errmsg, headers, data=None): @@ -42,7 +44,8 @@ class MyURLopener(urllib.URLopener): # Error 401 -- authentication required def http_error_401(self, url, fp, errcode, errmsg, headers, data=None): - raise IOError(('http error', errcode, "Authentication required ", headers)) + raise IOError( + ('http error', errcode, "Authentication required ", headers)) def http_error_default(self, url, fp, errcode, errmsg, headers): if fp: @@ -54,10 +57,13 @@ class MyURLopener(urllib.URLopener): urllib._urlopener = MyURLopener() # Fake headers to pretend this is a real browser -_user_agent = "Mozilla/5.0 (X11; U; Linux 2.6 i686; en) Gecko/20001221 Firefox/2.0.0" +_user_agent = "Mozilla/5.0 (X11; U; Linux 2.6 i686; en)" +" Gecko/20001221 Firefox/2.0.0" urllib._urlopener.addheaders[0] = ('User-Agent', _user_agent) _x_user_agent = "bookmarks_db (Python %d.%d.%d; urllib/%s)" % ( - sys.version_info[0], sys.version_info[1], sys.version_info[2], urllib.__version__) + sys.version_info[0], sys.version_info[1], + sys.version_info[2], urllib.__version__ +) urllib._urlopener.addheader('X-User-Agent', _x_user_agent) urllib._urlopener.addheader('Referer', '') @@ -70,12 +76,14 @@ urllib._urlopener.addheader('Connection', 'close') urllib_ftpwrapper = urllib.ftpwrapper ftpcache_key = None + class myftpwrapper(urllib_ftpwrapper): def __init__(self, user, passwd, host, port, dirs): urllib_ftpwrapper.__init__(self, user, passwd, host, port, dirs) global ftpcache_key ftpcache_key = (user, host, port, '/'.join(dirs)) + urllib.ftpwrapper = myftpwrapper @@ -91,7 +99,8 @@ class robot_urllib(robot_base): fname, headers = urllib.urlretrieve(url) finally: if accept_charset and bookmark.charset: - del urllib._urlopener.addheaders[-1] # Remove Accept-Charset + # Remove Accept-Charset + del urllib._urlopener.addheaders[-1] infile = open(fname, 'rb') content = infile.read() @@ -105,7 +114,8 @@ class robot_urllib(robot_base): except IOError as e: if (e[0] == "http error") and (e[1] == -1): error = None - bookmark.no_error = "The server did not return any header - it is not an error, actually" + bookmark.no_error = "The server did not return any header - " + "it is not an error, actually" self.log(' no headers: %s' % bookmark.no_error) else: error = get_error(e)