From: Oleg Broytman <phd@phdru.name>
Date: Sun, 11 Dec 2011 03:43:17 +0000 (+0000)
Subject: Debug HTML parsers.
X-Git-Tag: v4.5.3~29
X-Git-Url: https://git.phdru.name/?a=commitdiff_plain;h=eeffe06ce42763abfff7797ab0ceeedbac4b1669;p=bookmarks_db.git

Debug HTML parsers.


git-svn-id: file:///home/phd/archive/SVN/bookmarks_db/trunk@347 fdd5c36f-1aea-0310-aeeb-c58d7e2b6c23
---

diff --git a/parse_html/bkmk_parse_html.py b/parse_html/bkmk_parse_html.py
index a6b3d92..e1d4710 100644
--- a/parse_html/bkmk_parse_html.py
+++ b/parse_html/bkmk_parse_html.py
@@ -20,12 +20,6 @@ DEFAULT_CHARSET = "cp1251" # Stupid default for Russian Cyrillic
 
 parsers = []
 
-# Statistics by parser - successfully parsed HTML pages:
-# 4358 beautifulsoup
-# 4310 htmlparser
-# 4307 html5
-# 4250 lxml
-
 try:
    from . import bkmk_ph_beautifulsoup
 except ImportError:
@@ -92,6 +86,9 @@ def recode_entities(title, charset):
    return ''.join(output)
 
 
+import os
+BKMK_DEBUG_HTML_PARSERS = os.environ.get("BKMK_DEBUG_HTML_PARSERS")
+
 def parse_html(filename, charset=None, log=None):
    if not parsers:
        return None
@@ -109,7 +106,8 @@ def parse_html(filename, charset=None, log=None):
          charsets.remove(charset)
       charsets.insert(0, charset)
 
-   #_parsers = []
+   if BKMK_DEBUG_HTML_PARSERS:
+      _parsers = []
    for p in parsers:
       parser = None
       for c in charsets:
@@ -119,20 +117,25 @@ def parse_html(filename, charset=None, log=None):
             pass
          else:
             if parser:
-               #if log: log("   Parser %s: ok" % p.__module__)
-               #_parsers.append(parser)
+               if BKMK_DEBUG_HTML_PARSERS:
+                  if log: log("   Parser %s: ok" % p.__module__)
+                  _parsers.append((p, parser))
                break
       else:
          if log: log("   Parser %s: fail" % p.__module__)
-      if parser:
+      if not BKMK_DEBUG_HTML_PARSERS and parser:
          break
 
-   #if not _parsers:
-   if not parser:
+   if BKMK_DEBUG_HTML_PARSERS:
+      if not _parsers:
+         if log: log("   All parsers have failed")
+         return None
+   elif not parser:
        if log: log("   All parsers have failed")
        return None
 
-   #parser = _parsers[0]
+   if BKMK_DEBUG_HTML_PARSERS:
+      p, parser = _parsers[0]
    if log: log("   Using %s" % p.__module__)
 
    converted_title = title = parser.title