Fix(Py3): Fix HTML parsers

diff --git a/parse_html/bkmk_parse_html.py b/parse_html/bkmk_parse_html.py
index 85aeb88e4b63bef832f7f67871cb32e54e57a3de..be5daab26e85ec47620838f506b49085c392e78f 100644 (file)
--- a/parse_html/bkmk_parse_html.py
+++ b/parse_html/bkmk_parse_html.py
@@ -168,7 +168,7 @@ def parse_html(html_text, charset=None, log=None):
     #            parser.charset = 'ascii'
 
     converted_title = title = parser.title
-    if title and (not parser.charset):
+    if title and isinstance(title, bytes) and (not parser.charset):
         try:
             title.decode("ascii")
         except UnicodeDecodeError:

diff --git a/parse_html/bkmk_ph_beautifulsoup4.py b/parse_html/bkmk_ph_beautifulsoup4.py
index 1095ebce8d510c22df2c9768baed3c27777e1909..6549683edcb094a4282d89a6846aba81286016a0 100644 (file)
--- a/parse_html/bkmk_ph_beautifulsoup4.py
+++ b/parse_html/bkmk_ph_beautifulsoup4.py
@@ -11,18 +11,29 @@ __license__ = "GNU GPL"
 __all__ = ['parse_html']
 
 
+import warnings
+
 from bs4 import BeautifulSoup
 
 from .bkmk_ph_util import HTMLParser
 from compat import string_type
 
+warnings.filterwarnings(
+    'ignore', 'No parser was explicitly specified')
+warnings.filterwarnings(
+    'ignore',
+    "It looks like you're parsing an XML document using an HTML parser.")
+
 universal_charset = "utf-8"
 DEFAULT_CHARSET = "cp1251"  # Stupid default for Russian Cyrillic
 
 
 def _parse_html(html_text, charset):
     try:
-        return BeautifulSoup(html_text, from_encoding=charset)
+        if isinstance(html_text, bytes):
+            return BeautifulSoup(html_text, from_encoding=charset)
+        else:
+            return BeautifulSoup(html_text)
     except TypeError:
         return None
 

diff --git a/parse_html/bkmk_ph_html5.py b/parse_html/bkmk_ph_html5.py
index 68c1ababbb106f33a14d08e3c51d9e8d19e17573..1fabd82166611fcf5c480463296e07390c906be1 100644 (file)
--- a/parse_html/bkmk_ph_html5.py
+++ b/parse_html/bkmk_ph_html5.py
@@ -17,8 +17,11 @@ from .bkmk_ph_util import HTMLParser
 
 def parse_html(html_text, charset=None, log=None):
     parser = HTML5Parser()
-    html_tree = parser.parse(
-        html_text, encoding=charset, parseMeta=bool(charset))
+    if isinstance(html_text, bytes):
+        html_tree = parser.parse(
+            html_text, encoding=charset, parseMeta=bool(charset))
+    else:
+        html_tree = parser.parse(html_text)
 
     html = None
     if hasattr(html_tree, 'childNodes'):

diff --git a/parse_html/bkmk_ph_htmlparser.py b/parse_html/bkmk_ph_htmlparser.py
index b90618f1856d37a99bd78e7931421aa67e6d45b3..fd7b687d102f131928f675e0132c9a55dd01ef49 100644 (file)
--- a/parse_html/bkmk_ph_htmlparser.py
+++ b/parse_html/bkmk_ph_htmlparser.py
@@ -11,7 +11,10 @@ __license__ = "GNU GPL"
 __all__ = ['parse_html']
 
 
-from HTMLParser import HTMLParseError
+try:
+    from HTMLParser import HTMLParseError
+except ImportError:
+    class HTMLParseError(Exception): pass
 from m_lib.net.www.html import HTMLParser as _HTMLParser
 
 

diff --git a/parse_html/bkmk_ph_lxml.py b/parse_html/bkmk_ph_lxml.py
index 03dd6f4c0d90bb60627a442d1dfb743eb7a07961..a02de919fb3cfef336b287ceb753e2d3d0cd980f 100644 (file)
--- a/parse_html/bkmk_ph_lxml.py
+++ b/parse_html/bkmk_ph_lxml.py
@@ -11,15 +11,32 @@ __license__ = "GNU GPL"
 __all__ = ['parse_html']
 
 
-from lxml.html import fromtring
+import re
+from lxml.html import fromstring
 from .bkmk_ph_util import HTMLParser
 
 
 def parse_html(html_text, charset=None, log=None):
-    html_tree = fromtring(html_text)
-
-    if html_tree.getroot() is None:
-        return None
+    try:
+        html_tree = fromstring(html_text)
+    except ValueError as e:
+        if e.args[0].startswith(
+            'Unicode strings with encoding declaration are not supported.'
+            ' Please use bytes input'
+        ):
+            if not charset:
+                match = re.search(
+                    '<\\?xml version="(\\d|.)+" encoding="([^"]+)"\\?>',
+                    html_text, re.U)
+                if match:
+                    charset = match.group(2)
+            if charset:
+                html_text = html_text.encode(charset)
+                html_tree = fromstring(html_text)
+            else:
+                return None
+        else:
+            raise
 
     title = html_tree.findtext('head/title')
     if title is None:

diff --git a/parse_html/bkmk_ph_util.py b/parse_html/bkmk_ph_util.py
index 0fd09f28fa0aa35712b90dc46fca447a2db7f8a6..1aeb306a56c40c09ef0b5c9d827cf6d2dca9abdf 100644 (file)
--- a/parse_html/bkmk_ph_util.py
+++ b/parse_html/bkmk_ph_util.py
@@ -10,10 +10,10 @@ __license__ = "GNU GPL"
 __all__ = ['HTMLParser']
 
 
-from HTMLParser import HTMLParser
+from m_lib.net.www.html import HTMLParser as _HTMLParser
 
 
-class HTMLParser(HTMLParser):
+class HTMLParser(_HTMLParser):
     def __init__(self, charset, meta_charset, title, refresh, icon):
         object.__init__(self)
         self.charset = charset