reloc_dict = {
- 301: "perm.",
+ 301: "perm1.",
302: "temp2.",
303: "temp3.",
307: "temp7.",
+ 308: "temp8.",
"html": "html"
}
if headers:
try:
content_type = headers["Content-Type"]
- self.log(" Content-Type: %s" % content_type)
+ self.log(" Content-Type : %s" % content_type)
if content_type is None:
if 'html' in content.lower():
content_type = 'text/html'
except (ValueError, IndexError):
charset = None
self.log(" no charset in Content-Type header")
+ is_html = False
for ctype in ("text/html", "application/xhtml+xml"):
if content_type.startswith(ctype):
- html = True
+ is_html = True
break
- else:
- html = False
- if html:
- parser = parse_html(content, charset, self.log)
+ content_stripped = content.strip()
+ if content_stripped and is_html:
+ parser = parse_html(content_stripped, charset, self.log)
if parser:
bookmark.real_title = parser.title
icon = parser.icon
% (url, timeout)
)
+ if not content_stripped:
+ self.log(" empty response, no content")
+ if not is_html:
+ self.log(" not html")
except KeyError as key:
self.log(" no header: %s" % key)
return 1
def set_redirect(self, bookmark, errcode, newurl):
- bookmark.moved = "(%s) to %s" % (reloc_dict[errcode], newurl)
- self.log(' Moved: %s' % bookmark.moved)
+ bookmark.moved = moved = "(%s) to %s" % (reloc_dict[errcode], newurl)
+ try:
+ moved.encode('ascii')
+ except UnicodeEncodeError:
+ try:
+ moved = moved.encode(bookmark.charset)
+ except (LookupError, TypeError, UnicodeEncodeError):
+ moved = moved.encode('utf-8')
+ self.log(' Moved: %s' % moved)
def finish_check_url(self, bookmark):
start = self.start