Feat: Remove some HTML parsers

EtreeTidy is outdated and buggy.
html5 is outdated.

diff --git a/parse_html/bkmk_parse_html.py b/parse_html/bkmk_parse_html.py
index 7764303a4e754715a86a3a12e5d0b48d1d5acb8c..4d5f9e6d11f06ef0b587acca2b608b6443007795 100644 (file)
--- a/parse_html/bkmk_parse_html.py
+++ b/parse_html/bkmk_parse_html.py
@@ -24,14 +24,6 @@ from compat import unicode, unichr
 DEFAULT_CHARSET = "cp1251"  # Stupid default for Russian Cyrillic
 parsers = []
 
-try:
-    from . import bkmk_ph_beautifulsoup4
-except ImportError:
-    pass
-else:
-    bkmk_ph_beautifulsoup4.DEFAULT_CHARSET = DEFAULT_CHARSET
-    parsers.append(bkmk_ph_beautifulsoup4.parse_html)
-
 try:
     from . import bkmk_ph_beautifulsoup
 except ImportError:
@@ -41,33 +33,26 @@ else:
     parsers.append(bkmk_ph_beautifulsoup.parse_html)
 
 try:
-    from . import bkmk_ph_html5
+    from . import bkmk_ph_beautifulsoup4
 except ImportError:
     pass
 else:
-    parsers.append(bkmk_ph_html5.parse_html)
+    bkmk_ph_beautifulsoup4.DEFAULT_CHARSET = DEFAULT_CHARSET
+    parsers.append(bkmk_ph_beautifulsoup4.parse_html)
 
 try:
-    from . import bkmk_ph_lxml
+    from . import bkmk_ph_htmlparser
 except ImportError:
     pass
 else:
-    parsers.append(bkmk_ph_lxml.parse_html)
+    parsers.append(bkmk_ph_htmlparser.parse_html)
 
 try:
-    from . import bkmk_ph_htmlparser
+    from . import bkmk_ph_lxml
 except ImportError:
     pass
 else:
-    parsers.append(bkmk_ph_htmlparser.parse_html)
-
-# ElementTidy often segfaults
-# try:
-#     from . import bkmk_ph_etreetidy
-# except ImportError:
-#     pass
-# else:
-#     parsers.append(bkmk_ph_etreetidy.parse_html)
+    parsers.append(bkmk_ph_lxml.parse_html)
 
 universal_charset = "utf-8"
 entity_re = re.compile("(&\\w+;)")

diff --git a/parse_html/bkmk_ph_etreetidy.py b/parse_html/bkmk_ph_etreetidy.py
deleted file mode 100644 (file)
index eadcca3..0000000
--- a/parse_html/bkmk_ph_etreetidy.py
+++ /dev/null
@@ -1,72 +0,0 @@
-"""HTML Parser using ElementTree+TidyLib.
-
-This file is a part of Bookmarks database and Internet robot.
-
-"""
-
-__author__ = "Oleg Broytman <phd@phdru.name>"
-__copyright__ = "Copyright (C) 2010-2023 PhiloSoft Design"
-__license__ = "GNU GPL"
-
-__all__ = ['parse_html']
-
-
-from elementtidy import TidyHTMLTreeBuilder
-from .bkmk_ph_util import HTMLParser
-
-
-def parse_html(html_text, charset=None, log=None):
-    if not html_text:
-        return None
-    try:
-        html_tree = TidyHTMLTreeBuilder.parseString(html_text)
-    except:
-        return None
-
-    XHTML = "{http://www.w3.org/1999/xhtml}"
-
-    for elem in html_tree.getiterator():
-        if elem.tag.startswith(XHTML):
-            elem.tag = elem.tag[len(XHTML):]
-
-    title = html_tree.findtext('head/title')
-    if title is None:
-        title = html_tree.findtext('title')
-
-    meta = html_tree.findall('head/meta')
-    for m in meta:
-        if m.get('http-equiv', '').lower() == 'content-type':
-            meta_content = m.get("content")
-            if meta_content:
-                try:
-                    meta_charset = \
-                        meta_content.lower().split('charset=')[1].split(';')[0]
-                    break
-                except IndexError:
-                    meta_charset = False
-        elif m.get('charset', ''):
-            meta_charset = m.get('charset').lower()
-            break
-    else:
-        meta_charset = False
-
-    #if title and (charset or meta_charset):
-    #    title = title.encode(charset or meta_charset)
-
-    for m in meta:
-        if m.get('http-equiv', '').lower() == 'refresh':
-            refresh = m.get("content")
-            break
-    else:
-        refresh = None
-
-    for link in html_tree.findall('head/link'):
-        if link.get('rel', '').lower() in ('icon', 'shortcut icon'):
-            icon = link.get("href")
-            break
-    else:
-        icon = None
-
-    if (title is None) and (refresh is None) and (icon is None):
-        return None
-    return HTMLParser(charset, meta_charset, title, refresh, icon)

diff --git a/parse_html/bkmk_ph_html5.py b/parse_html/bkmk_ph_html5.py
deleted file mode 100644 (file)
index d973b72..0000000
--- a/parse_html/bkmk_ph_html5.py
+++ /dev/null
@@ -1,109 +0,0 @@
-"""HTML Parser using html5
-
-This file is a part of Bookmarks database and Internet robot.
-
-"""
-
-__author__ = "Oleg Broytman <phd@phdru.name>"
-__copyright__ = "Copyright (C) 2010-2023 PhiloSoft Design"
-__license__ = "GNU GPL"
-
-__all__ = ['parse_html']
-
-
-from html5lib import HTMLParser as HTML5Parser
-from .bkmk_ph_util import HTMLParser
-
-
-def parse_html(html_text, charset=None, log=None):
-    if not html_text:
-        return None
-    parser = HTML5Parser()
-    if isinstance(html_text, bytes):
-        html_tree = parser.parse(
-            html_text, encoding=charset, parseMeta=bool(charset))
-    else:
-        html_tree = parser.parse(html_text)
-
-    html = None
-    if hasattr(html_tree, 'childNodes'):
-        for node in html_tree.childNodes:
-            # Skip DocType element
-            if (node.name == 'html') and (node.type != 3):
-                html = node
-                break
-
-    if not html:
-        return None
-
-    for node in html.childNodes:
-        if node.name == 'head':
-            head = node
-            break
-    else:
-        head = None
-
-    meta_charset = False
-    title = None
-    refresh = None
-    icon = None
-
-    if head:
-        for node in head.childNodes:
-            if node.name == 'title':
-                if node.childNodes:
-                    title = node.childNodes[0].value
-                    break
-                else:
-                    title = ''
-
-        for node in head.childNodes:
-            if (node.name == 'meta') and \
-                    ('http-equiv' in node.attributes) and \
-                    (node.attributes['http-equiv'] == 'content-type'):
-                meta_content = node.attributes['content']
-                if meta_content:
-                    try:
-                        meta_charset = \
-                            meta_content.lower().split('charset=')[1].\
-                            split(';')[0]
-                    except IndexError:
-                        meta_charset = False
-                    else:
-                        break
-            elif (node.name == 'meta') and ('charset' in node.attributes):
-                meta_charset = node.attributes['charset'].lower()
-                break
-
-        if not charset:
-            charset = parser.tokenizer.stream.charEncoding[0]
-
-        #if title and (charset or meta_charset):
-        #    title = title.encode(charset or meta_charset)
-
-        for node in head.childNodes:
-            if node.name == 'meta' and \
-                    ('http-equiv' in node.attributes) and \
-                    (node.attributes['http-equiv'] == 'refresh'):
-                refresh = node.attributes['content']
-                break
-
-        for node in head.childNodes:
-            if node.name == 'link' and \
-                    ('rel' in node.attributes) and \
-                    (node.attributes['rel'] in ('icon', 'shortcut icon')):
-                icon = node.attributes['href']
-                break
-
-    else:
-        for node in html.childNodes:
-            if node.name == 'title':
-                if node.childNodes:
-                    title = node.childNodes[0].value
-                    break
-                else:
-                    title = ''
-
-    if (title is None) and (refresh is None) and (icon is None):
-        return None
-    return HTMLParser(charset, meta_charset, title, refresh, icon)