X-Git-Url: https://git.phdru.name/?a=blobdiff_plain;f=Robots%2Fparse_html.py;h=666c7071fd0072a8249b9e2bf8a0ba4cb4a6e3c6;hb=95b39d3b8b276f64c706c253c6ee643bf93725aa;hp=fc9514d005e5c02dce4fe4717c7e1732060d7cd6;hpb=b823c1b58711d6c4ed96fcb31de5cf348d8a6c1a;p=bookmarks_db.git

diff --git a/Robots/parse_html.py b/Robots/parse_html.py
index fc9514d..666c707 100755
--- a/Robots/parse_html.py
+++ b/Robots/parse_html.py
@@ -30,7 +30,10 @@ def recode_entities(title, charset):
    output = []
    for part in entity_re.split(title):
       if entity_re.match(part):
-         part = unichr(int(part[2:-1])).encode(charset, "replace")
+         try:
+            part = unichr(int(part[2:-1])).encode(charset)
+         except UnicodeEncodeError:
+            pass # Leave the entity as is
       output.append(part)
 
    return ''.join(output)
@@ -48,7 +51,7 @@ def parse_html(filename, charset=None, log=None):
       if parser:
          break
       else:
-         if log: log("Parser %s failed, trying next one." % p)
+         if log: log("Parser %s.%s failed, trying next one." % (p.__module__, p.__name__))
 
    title = parser.title
    if not parser.charset:
@@ -83,7 +86,10 @@ def parse_html(filename, charset=None, log=None):
       except LookupError:
          if log: log("   unknown charset: `%s' or `%s'" % (parser.charset, current_charset))
 
-   parser.title = recode_entities(title, current_charset)
+   title = recode_entities(title, current_charset)
+   title = title.replace('\r', '').replace('\n', ' ').strip()
+   if log: log("   final title    : %s" % title)
+   parser.title = title
    return parser