From: Oleg Broytman <phd@phdru.name>
Date: Wed, 11 Aug 2010 18:22:25 +0000 (+0000)
Subject: Added HTML Parser based on TidyHTMLTreeBuilder.
X-Git-Tag: v4.5.3~118
X-Git-Url: https://git.phdru.name/?a=commitdiff_plain;h=349d6e0241d43e42a257d6019972402116236ee7;p=bookmarks_db.git

Added HTML Parser based on TidyHTMLTreeBuilder.


git-svn-id: file:///home/phd/archive/SVN/bookmarks_db/trunk@258 fdd5c36f-1aea-0310-aeeb-c58d7e2b6c23
---

diff --git a/Robots/parse_html.py b/Robots/parse_html.py
index 8ffca5c..cc7655b 100755
--- a/Robots/parse_html.py
+++ b/Robots/parse_html.py
@@ -11,6 +11,14 @@ universal_charset = "utf-8"
 DEFAULT_CHARSET = "cp1251" # Stupid default for Russian Cyrillic
 
 parsers = []
+
+try:
+   import parse_html_etreetidy
+except ImportError:
+   pass
+else:
+   parsers.append(parse_html_etreetidy.parse_html)
+
 try:
    import parse_html_beautifulsoup
    parse_html_beautifulsoup.DEFAULT_CHARSET = DEFAULT_CHARSET
diff --git a/Robots/parse_html_etreetidy.py b/Robots/parse_html_etreetidy.py
new file mode 100644
index 0000000..65d42ae
--- /dev/null
+++ b/Robots/parse_html_etreetidy.py
@@ -0,0 +1,52 @@
+"""
+    HTML Parser using ElementTree+TidyLib.
+
+    Written by Broytman. Copyright (C) 2010 PhiloSoft Design
+"""
+
+from elementtidy import TidyHTMLTreeBuilder
+from parse_html_util import HTMLParser
+
+
+def parse_html(filename, charset=None, log=None):
+    try:
+        html_tree = TidyHTMLTreeBuilder.parse(filename)
+    except:
+        return None
+
+    XHTML = "{http://www.w3.org/1999/xhtml}"
+
+    for elem in html_tree.getiterator():
+        if elem.tag.startswith(XHTML):
+            elem.tag = elem.tag[len(XHTML):]
+
+    meta = html_tree.findall('head/meta')
+    for m in meta:
+        if m.get('http-equiv', '').lower() == 'content-type':
+            meta_content = m.get("content")
+            if meta_content:
+                meta_charset = \
+                    meta_content.lower().split('charset=')[1].split(';')[0]
+                break
+    else:
+        meta_charset = False
+
+    title = html_tree.findtext('head/title')
+    if title and (charset or meta_charset):
+        title = title.encode(charset or meta_charset)
+
+    for m in meta:
+        if m.get('http-equiv', '').lower() == 'refresh':
+            refresh = m.get("content")
+            break
+    else:
+        refresh = None
+
+    for link in html_tree.findall('head/link'):
+        if link.get('rel', '').lower() in ('icon', 'shortcut icon'):
+            icon = link.get("href")
+            break
+    else:
+        icon = None
+
+    return HTMLParser(charset, meta_charset, title, refresh, icon)