Do not parse meta charset if there is HTTP charset.

Find html in tree.childNodes skipping DocType.
Use meta charset if there is no HTTP charset.

git-svn-id: file:///home/phd/archive/SVN/bookmarks_db/trunk@267 fdd5c36f-1aea-0310-aeeb-c58d7e2b6c23

diff --git a/Robots/parse_html_html5.py b/Robots/parse_html_html5.py
index fcc7e158a20e2e0a6f35aa8840cbb197dd1c8755..6255825a3b491d06fb0df847937ed5e638587f36 100644 (file)
--- a/Robots/parse_html_html5.py
+++ b/Robots/parse_html_html5.py
@@ -9,14 +9,22 @@ from parse_html_util import HTMLParser
 
 
 def parse_html(filename, charset=None, log=None):
+    parser = HTML5Parser()
     fp = open(filename)
-    html_tree = HTML5Parser().parse(fp, charset)
+    parser._parse(fp, encoding=charset, parseMeta=bool(charset))
     fp.close()
+    html_tree = parser.tree.getDocument()
 
-    if not html_tree.childNodes:
+    for node in html_tree.childNodes:
+        if (node.name == 'html') and (node.type != 3): # Skip DocType element
+            html = node
+            break
+    else:
+        html = None
+
+    if not html:
         return None
 
-    html = html_tree.childNodes[-1]
     for node in html.childNodes:
         if node.name == 'head':
             head = node
@@ -52,6 +60,9 @@ def parse_html(filename, charset=None, log=None):
                 else:
                     title = ''
 
+        if not charset:
+            charset = parser.tokenizer.stream.charEncoding[0]
+
         if title and (charset or meta_charset):
             title = title.encode(charset or meta_charset)