Feat(Python3): `<>` -> `!=`

[bookmarks_db.git] / parse_html / bkmk_parse_html.py

diff --git a/parse_html/bkmk_parse_html.py b/parse_html/bkmk_parse_html.py
index 42cb5ce1575abf2f0e1eb073943ca62240bcccb2..2e412ad84dc69756662c215e287e120dad66f72a 100644 (file)
--- a/parse_html/bkmk_parse_html.py
+++ b/parse_html/bkmk_parse_html.py
@@ -1,16 +1,14 @@
 """HTML Parsers
 
 This file is a part of Bookmarks database and Internet robot.
+
 """
 
-__version__ = "$Revision$"[11:-2]
-__revision__ = "$Id$"[5:-2]
-__date__ = "$Date$"[7:-2]
 __author__ = "Oleg Broytman <phd@phdru.name>"
-__copyright__ = "Copyright (C) 1997-2011 PhiloSoft Design"
+__copyright__ = "Copyright (C) 1997-2017 PhiloSoft Design"
 __license__ = "GNU GPL"
 
-__all__ = ['parse_html', 'universal_charset']
+__all__ = ['parse_html', 'parse_filename', 'universal_charset']
 
 
 import codecs
@@ -20,12 +18,6 @@ DEFAULT_CHARSET = "cp1251" # Stupid default for Russian Cyrillic
 
 parsers = []
 
-# Statistics by parser - successfully parsed HTML pages:
-# 4358 beautifulsoup
-# 4310 htmlparser
-# 4307 html5
-# 4250 lxml
-
 try:
    from . import bkmk_ph_beautifulsoup
 except ImportError:
@@ -35,25 +27,25 @@ else:
    parsers.append(bkmk_ph_beautifulsoup.parse_html)
 
 try:
-   from . import bkmk_ph_htmlparser
+   from . import bkmk_ph_html5
 except ImportError:
    pass
 else:
-    parsers.append(bkmk_ph_htmlparser.parse_html)
+   parsers.append(bkmk_ph_html5.parse_html)
 
 try:
-   from . import bkmk_ph_html5
+   from . import bkmk_ph_lxml
 except ImportError:
    pass
 else:
-   parsers.append(bkmk_ph_html5.parse_html)
+    parsers.append(bkmk_ph_lxml.parse_html)
 
 try:
-   from . import bkmk_ph_lxml
+   from . import bkmk_ph_htmlparser
 except ImportError:
    pass
 else:
-    parsers.append(bkmk_ph_lxml.parse_html)
+    parsers.append(bkmk_ph_htmlparser.parse_html)
 
 # ElementTidy often segfaults
 #try:
@@ -92,7 +84,10 @@ def recode_entities(title, charset):
    return ''.join(output)
 
 
-def parse_html(filename, charset=None, log=None):
+import os
+BKMK_DEBUG_HTML_PARSERS = os.environ.get("BKMK_DEBUG_HTML_PARSERS")
+
+def parse_html(html_text, charset=None, log=None):
    if not parsers:
        return None
 
@@ -109,31 +104,37 @@ def parse_html(filename, charset=None, log=None):
          charsets.remove(charset)
       charsets.insert(0, charset)
 
-   #_parsers = []
+   if BKMK_DEBUG_HTML_PARSERS:
+      _parsers = []
    for p in parsers:
       parser = None
       for c in charsets:
          try:
-            parser = p(filename, c, log)
+            parser = p(html_text, c, log)
          except UnicodeError:
             pass
          else:
             if parser:
-               if log: log("   Parser %s: ok" % p.__module__)
-               #_parsers.append(parser)
+               if BKMK_DEBUG_HTML_PARSERS:
+                  if log: log("   Parser %s: ok" % p.__module__)
+                  _parsers.append((p, parser))
                break
       else:
          if log: log("   Parser %s: fail" % p.__module__)
-      if parser:
+      if not BKMK_DEBUG_HTML_PARSERS and parser:
          break
 
-   #if not _parsers:
-   if not parser:
-       if log: log("   All parser has failed")
+   if BKMK_DEBUG_HTML_PARSERS:
+      if not _parsers:
+         if log: log("   All parsers have failed")
+         return None
+   elif not parser:
+       if log: log("   All parsers have failed")
        return None
 
-   #parser = _parsers[0]
-   if log: log("   Using %s" % parser.__module__)
+   if BKMK_DEBUG_HTML_PARSERS:
+      p, parser = _parsers[0]
+   if log: log("   Using %s" % p.__module__)
 
    converted_title = title = parser.title
    if title and (not parser.charset):
@@ -146,23 +147,23 @@ def parse_html(filename, charset=None, log=None):
       parser.charset = parser.charset.lower().replace("windows-", "cp")
 
    if title and parser.charset and (
-         (parser.charset <> universal_charset) or
-         ((not charset) or (charset <> parser.charset))):
+         (parser.charset != universal_charset) or
+         ((not charset) or (charset != parser.charset))):
       try:
          if parser.meta_charset:
             if log: log("   META charset   : %s" % parser.charset)
-         elif (not charset) or (charset <> parser.charset):
+         elif (not charset) or (charset != parser.charset):
             if log: log("   guessed charset: %s" % parser.charset)
          #if log: log("   current charset: %s" % universal_charset)
          if log: log("   title          : %s" % title)
-         if parser.charset <> universal_charset:
+         if parser.charset != universal_charset:
             try:
                converted_title = unicode(title, parser.charset).encode(universal_charset)
             except UnicodeError:
                if log: log("   incorrect conversion from %s, converting from %s" % (parser.charset, DEFAULT_CHARSET))
                converted_title = unicode(title, DEFAULT_CHARSET, "replace").encode(universal_charset, "replace")
                parser.charset = DEFAULT_CHARSET
-         if log and (converted_title <> title): log("   converted title: %s" % converted_title)
+         if log and (converted_title != title): log("   converted title: %s" % converted_title)
       except LookupError:
          if log: log("   unknown charset: '%s'" % parser.charset)
    else:
@@ -172,7 +173,7 @@ def parse_html(filename, charset=None, log=None):
       final_title = recode_entities(converted_title, universal_charset)
       parts = [s.strip() for s in final_title.replace('\r', '').split('\n')]
       final_title = ' '.join([s for s in parts if s])
-      if log and (final_title <> converted_title): log("   final title    : %s" % final_title)
+      if log and (final_title != converted_title): log("   final title    : %s" % final_title)
       parser.title = final_title
 
    icon = parser.icon
@@ -183,3 +184,11 @@ def parse_html(filename, charset=None, log=None):
            if parser.charset:
                parser.icon = icon.encode(parser.charset)
    return parser
+
+def parse_filename(filename, charset=None, log=None):
+    fp = open(filename, 'r')
+    try:
+        parser = parse_html(fp.read(), charset=charset, log=log)
+    finally:
+        fp.close()
+    return parser