import sys
import socket
import time
-import urllib
-from urlparse import urljoin
+try:
+ from urllib.parse import splittype, splithost, splittag, urljoin
+except ImportError:
+ from urllib import splittype, splithost, splittag
+ from urlparse import urljoin
from m_lib.md5wrapper import md5wrapper
from m_lib.net.www.util import parse_time
self.start = int(time.time())
bookmark.icon = None
- url_type, url_rest = urllib.splittype(bookmark.href)
- url_host, url_path = urllib.splithost(url_rest)
- url_path, url_tag = urllib.splittag(url_path) # noqa: E221
- # multiple spaces before operator
+ url_type, url_rest = splittype(bookmark.href)
+ url_host, url_path = splithost(url_rest)
+ url_path, url_tag = splittag(url_path) # noqa: E221
+ # multiple spaces before operator
url = "%s://%s%s" % (url_type, url_host, url_path)
error, redirect_code, redirect_to, headers, content = \
md5 = md5wrapper()
if url_type == "ftp": # Pass welcome message through MD5
- md5.update(self.get_ftp_welcome())
+ ftp_welcome = self.get_ftp_welcome()
+ if not isinstance(ftp_welcome, bytes):
+ ftp_welcome = ftp_welcome.encode('utf-8')
+ md5.update(ftp_welcome)
- md5.update(content)
+ if isinstance(content, bytes):
+ md5.update(content)
+ else:
+ md5.update(content.encode('utf-8'))
bookmark.md5 = str(md5)
if headers:
try:
content_type = headers["Content-Type"]
self.log(" Content-Type: %s" % content_type)
+ if content_type is None:
+ if 'html' in content.lower():
+ content_type = 'text/html'
+ else:
+ content_type = 'text/plain'
+ self.log(" Set Content-Type to: %s"
+ % content_type)
try:
# extract charset from
# "text/html; foo; charset=UTF-8, bar; baz;"
icons[icon_url] = None
else:
content_type = icon_headers["Content-Type"]
- if content_type.startswith("application/") \
- or content_type.startswith("image/") \
- or content_type.startswith("text/plain"):
+ if content_type and (
+ content_type.startswith("application/")
+ or content_type.startswith("image/")
+ or content_type.startswith("text/plain")
+ ):
bookmark.icon_href = icon_url
self.log(" got icon : %s"
% content_type)
self.log(" non-image content type,"
" assume x-icon")
content_type = 'image/x-icon'
+ if not isinstance(icon_data, bytes):
+ icon_data = icon_data.encode('utf-8')
bookmark.icon = "data:%s;base64,%s" \
% (content_type, b64encode(icon_data))
icons[icon_url] = (content_type,